29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
35 #include "cpl_azure.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
42 #include <curl/curl.h>
52 #if LIBCURL_VERSION_NUM >= 0x071201
53 #define HAVE_CURLINFO_REDIRECT_URL
56 void VSICurlStreamingClearCache(
void );
58 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle,
const char* pszURL,
59 const char *
const* papszOptions);
60 struct curl_slist* VSICurlMergeHeaders(
struct curl_slist* poDest,
61 struct curl_slist* poSrcToDestroy );
75 unsigned int nGenerationAuthParameters = 0;
76 ExistStatus eExists = EXIST_UNKNOWN;
79 time_t nExpireTimestampLocal = 0;
81 bool bHasComputedFileSize =
false;
82 bool bIsDirectory =
false;
83 bool bS3LikeRedirect =
false;
89 bool bGotFileList =
false;
90 unsigned int nGenerationAuthParameters = 0;
94 struct WriteFuncStruct
96 char* pBuffer =
nullptr;
99 bool bIsInHeader =
false;
100 bool bMultiRange =
false;
105 bool bFoundContentRange =
false;
107 bool bDownloadHeaderOnly =
false;
108 bool bDetectRangeDownloadingError =
false;
112 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
113 void *pReadCbkUserData =
nullptr;
114 bool bInterrupted =
false;
116 #if LIBCURL_VERSION_NUM < 0x073600
120 bool bIsProxyConnectHeader =
false;
126 const GByte* pabyData =
nullptr;
128 size_t nTotalSize = 0;
130 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
131 size_t nitems,
void *instream )
133 PutData* poThis =
static_cast<PutData *
>(instream);
134 const size_t nSizeMax = size * nitems;
135 const size_t nSizeToWrite =
136 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
137 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
138 poThis->nOff += nSizeToWrite;
149 class VSICurlFilesystemHandler :
public VSIFilesystemHandler
153 struct FilenameOffsetPair
155 std::string filename_;
158 FilenameOffsetPair(
const std::string& filename,
160 filename_(filename), offset_(offset) {}
162 bool operator==(
const FilenameOffsetPair& other)
const
164 return filename_ == other.filename_ &&
165 offset_ == other.offset_;
168 struct FilenameOffsetPairHasher
170 std::size_t operator()(
const FilenameOffsetPair& k)
const
172 return std::hash<std::string>()(k.filename_) ^
173 std::hash<vsi_l_offset>()(k.offset_);
177 using RegionCacheType =
178 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
182 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
183 std::shared_ptr<std::string>>>::iterator,
184 FilenameOffsetPairHasher>>;
186 std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{};
187 RegionCacheType* GetRegionCache();
189 lru11::Cache<std::string, FileProp> oCacheFileProp;
191 int nCachedFilesInDirList = 0;
192 lru11::Cache<std::string, CachedDirList> oCacheDirList;
194 char** ParseHTMLFileList(
const char* pszFilename,
197 bool* pbGotFileList);
200 CPLMutex *hMutex =
nullptr;
202 virtual VSICurlHandle* CreateFileHandle(
const char* pszFilename);
203 virtual char** GetFileList(
const char *pszFilename,
205 bool* pbGotFileList);
207 void RegisterEmptyDir(
const CPLString& osDirname );
209 bool AnalyseS3FileList(
const CPLString& osBaseURL,
213 bool bIgnoreGlacierStorageClass,
214 bool& bIsTruncated );
216 void AnalyseSwiftFileList(
const CPLString& osBaseURL,
220 int nMaxFilesThisQuery,
225 static const char* GetOptionsStatic();
227 static bool IsAllowedFilename(
const char* pszFilename );
230 VSICurlFilesystemHandler();
231 ~VSICurlFilesystemHandler()
override;
234 const char *pszAccess,
235 bool bSetError )
override;
237 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
238 int nFlags )
override;
239 int Unlink(
const char *pszFilename )
override;
240 int Rename(
const char *oldpath,
const char *newpath )
override;
241 int Mkdir(
const char *pszDirname,
long nMode )
override;
242 int Rmdir(
const char *pszDirname )
override;
243 char **ReadDir(
const char *pszDirname )
override
244 {
return ReadDirEx(pszDirname, 0); }
245 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles )
override;
246 char **SiblingFiles(
const char *pszFilename )
override;
248 int HasOptimizedReadMultiRange(
const char* )
249 override {
return true; }
251 const char* GetActualURL(
const char* pszFilename)
override;
253 const char* GetOptions()
override;
255 char** GetFileMetadata(
const char * pszFilename,
const char* pszDomain,
258 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
259 bool* pbGotFileList );
260 void InvalidateDirContent(
const char *pszDirname );
262 virtual CPLString GetFSPrefix() {
return "/vsicurl/"; }
263 virtual bool AllowCachedDataFor(
const char* pszFilename);
265 std::shared_ptr<std::string> GetRegion(
const char* pszURL,
268 void AddRegion(
const char* pszURL,
273 bool GetCachedFileProp(
const char* pszURL,
274 FileProp& oFileProp );
275 void SetCachedFileProp(
const char* pszURL,
276 FileProp& oFileProp );
277 void InvalidateCachedData(
const char* pszURL );
279 CURLM *GetCurlMultiHandleFor(
const CPLString& osURL );
281 virtual void ClearCache();
282 virtual void PartialClearCache(
const char* pszFilename);
285 bool GetCachedDirList(
const char* pszURL,
286 CachedDirList& oCachedDirList );
287 void SetCachedDirList(
const char* pszURL,
288 CachedDirList& oCachedDirList );
289 bool ExistsInCacheDirList(
const CPLString& osDirname,
bool *pbIsDir );
303 VSICurlFilesystemHandler* poFS =
nullptr;
305 bool m_bCached =
true;
307 FileProp oFileProp{};
310 char* m_pszURL =
nullptr;
312 char **m_papszHTTPOptions =
nullptr;
315 int nBlocksToDownload = 1;
317 bool bStopOnInterruptUntilUninstall =
false;
318 bool bInterrupted =
false;
319 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
320 void *pReadCbkUserData =
nullptr;
323 double m_dfRetryDelay = 0.0;
327 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
338 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
340 bool m_bUseHead =
false;
342 int ReadMultiRangeSingleGet(
int nRanges,
void ** ppData,
344 const size_t* panSizes );
345 CPLString GetRedirectURLIfValid(
bool& bHasExpired);
348 virtual struct curl_slist* GetCurlHeaders(
const CPLString& ,
349 const struct curl_slist* )
351 virtual bool AllowAutomaticRedirection() {
return true; }
352 virtual bool CanRestartOnError(
const char*,
const char*,
bool ) {
return false; }
353 virtual bool UseLimitRangeGetInsteadOfHead() {
return false; }
354 virtual bool IsDirectoryFromExists(
const char* ,
int ) {
return false; }
355 virtual void ProcessGetFileSizeResult(
const char* ) {}
356 void SetURL(
const char* pszURL);
357 virtual bool Authenticate() {
return false; }
361 VSICurlHandle( VSICurlFilesystemHandler* poFS,
362 const char* pszFilename,
363 const char* pszURLIn =
nullptr );
364 ~VSICurlHandle()
override;
368 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
369 int ReadMultiRange(
int nRanges,
void ** ppData,
371 const size_t* panSizes )
override;
372 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
374 int Flush()
override;
375 int Close()
override;
377 bool IsKnownFileSize()
const {
return oFileProp.bHasComputedFileSize; }
378 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
379 virtual vsi_l_offset GetFileSize(
bool bSetError ) {
return GetFileSizeOrHeaders(bSetError,
false); }
380 bool Exists(
bool bSetError );
381 bool IsDirectory()
const {
return oFileProp.bIsDirectory; }
382 time_t GetMTime()
const {
return oFileProp.mTime; }
385 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
387 int bStopOnInterruptUntilUninstall );
388 int UninstallReadCbk();
390 const char *GetURL()
const {
return m_pszURL; }
397 class IVSIS3LikeFSHandler:
public VSICurlFilesystemHandler
403 const char* pszSource,
404 const char* pszTarget,
405 GDALProgressFunc pProgressFunc,
406 void *pProgressData);
407 virtual int MkdirInternal(
const char *pszDirname,
bool bDoStatCheck );
410 char** GetFileList(
const char *pszFilename,
412 bool* pbGotFileList )
override;
414 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
415 const char* pszURI,
bool bAllowNoObject) = 0;
417 virtual int CopyObject(
const char *oldpath,
const char *newpath,
420 IVSIS3LikeFSHandler() =
default;
423 int Unlink(
const char *pszFilename )
override;
424 int Mkdir(
const char *pszDirname,
long nMode )
override;
425 int Rmdir(
const char *pszDirname )
override;
426 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
427 int nFlags )
override;
428 int Rename(
const char *oldpath,
const char *newpath )
override;
430 virtual int DeleteObject(
const char *pszFilename );
432 virtual const char* GetDebugKey()
const = 0;
434 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
435 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
437 bool Sync(
const char* pszSource,
const char* pszTarget,
438 const char*
const * papszOptions,
439 GDALProgressFunc pProgressFunc,
441 char*** ppapszOutputs )
override;
443 VSIDIR* OpenDir(
const char *pszPath,
int nRecurseDepth,
444 const char*
const *papszOptions)
override;
447 virtual CPLString InitiateMultipartUpload(
448 const std::string& osFilename,
449 IVSIS3LikeHandleHelper *poS3HandleHelper,
451 double dfRetryDelay);
454 const std::string& osUploadID,
455 const void* pabyBuffer,
457 IVSIS3LikeHandleHelper *poS3HandleHelper,
459 double dfRetryDelay);
460 virtual bool CompleteMultipart(
const CPLString& osFilename,
462 const std::vector<CPLString>& aosEtags,
463 IVSIS3LikeHandleHelper *poS3HandleHelper,
465 double dfRetryDelay);
466 virtual bool AbortMultipart(
const CPLString& osFilename,
468 IVSIS3LikeHandleHelper *poS3HandleHelper,
470 double dfRetryDelay);
477 class IVSIS3LikeHandle:
public VSICurlHandle
482 bool UseLimitRangeGetInsteadOfHead()
override {
return true; }
483 bool IsDirectoryFromExists(
const char* pszVerb,
484 int response_code )
override
487 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
490 void ProcessGetFileSizeResult(
const char* pszContent )
override
492 oFileProp.bIsDirectory = strstr(pszContent,
"ListBucketResult") !=
nullptr;
496 IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
497 const char* pszFilename,
498 const char* pszURLIn ) :
499 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
500 ~IVSIS3LikeHandle()
override {}
511 IVSIS3LikeFSHandler *m_poFS =
nullptr;
513 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
514 bool m_bUseChunked =
false;
517 int m_nBufferOff = 0;
518 int m_nBufferSize = 0;
519 bool m_bClosed =
false;
520 GByte *m_pabyBuffer =
nullptr;
522 int m_nPartNumber = 0;
523 std::vector<CPLString> m_aosEtags{};
524 bool m_bError =
false;
526 CURLM *m_hCurlMulti =
nullptr;
527 CURL *m_hCurl =
nullptr;
528 const void *m_pBuffer =
nullptr;
530 size_t m_nChunkedBufferOff = 0;
531 size_t m_nChunkedBufferSize = 0;
532 size_t m_nWrittenInPUT = 0;
535 double m_dfRetryDelay = 0.0;
536 WriteFuncStruct m_sWriteFuncHeaderData{};
539 bool DoSinglePartPUT();
541 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
542 size_t nitems,
void *instream );
543 size_t WriteChunked(
const void *pBuffer,
544 size_t nSize,
size_t nMemb );
545 int FinishChunkedTransfer();
547 void InvalidateParentDirectory();
550 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
551 const char* pszFilename,
552 IVSIS3LikeHandleHelper* poS3HandleHelper,
554 ~VSIS3WriteHandle()
override;
558 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
559 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
561 int Close()
override;
563 bool IsOK() {
return m_bUseChunked || m_pabyBuffer !=
nullptr; }
576 VSICurlFilesystemHandler* m_poFS =
nullptr;
581 int m_nBufferOff = 0;
582 int m_nBufferSize = 0;
583 int m_nBufferOffReadCallback = 0;
584 bool m_bClosed =
false;
585 GByte *m_pabyBuffer =
nullptr;
586 bool m_bError =
false;
588 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
589 size_t nitems,
void *instream );
590 virtual bool Send(
bool bIsLastBlock) = 0;
593 VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
594 const char* pszFSPrefix,
595 const char* pszFilename,
597 virtual ~VSIAppendWriteHandle();
601 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
602 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
604 int Close()
override;
606 bool IsOK() {
return m_pabyBuffer !=
nullptr; }
613 struct CurlRequestHelper
615 WriteFuncStruct sWriteFuncData{};
616 WriteFuncStruct sWriteFuncHeaderData{};
617 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
620 ~CurlRequestHelper();
621 long perform(CURL* hCurlHandle,
622 struct curl_slist* headers,
623 VSICurlFilesystemHandler *poFS,
624 IVSIS3LikeHandleHelper *poS3HandleHelper);
631 class NetworkStatisticsLogger
633 static int gnEnabled;
634 static NetworkStatisticsLogger gInstance;
636 NetworkStatisticsLogger() =
default;
638 std::mutex m_mutex{};
647 GIntBig nGETDownloadedBytes = 0;
649 GIntBig nPOSTDownloadedBytes = 0;
650 GIntBig nPOSTUploadedBytes = 0;
653 enum class ContextPathType
660 struct ContextPathItem
662 ContextPathType eType;
665 ContextPathItem(ContextPathType eTypeIn,
const CPLString& osNameIn):
666 eType(eTypeIn), osName(osNameIn) {}
668 bool operator< (
const ContextPathItem& other )
const
670 if(
static_cast<int>(eType) <
static_cast<int>(other.eType) )
672 if(
static_cast<int>(eType) >
static_cast<int>(other.eType) )
674 return osName < other.osName;
681 std::map<ContextPathItem, Stats> children{};
687 std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
689 static void ReadEnabled();
691 std::vector<Counters*> GetCountersForContext();
695 static inline bool IsEnabled()
701 return gnEnabled == TRUE;
704 static void EnterFileSystem(
const char* pszName);
706 static void LeaveFileSystem();
708 static void EnterFile(
const char* pszName);
710 static void LeaveFile();
712 static void EnterAction(
const char* pszName);
714 static void LeaveAction();
716 static void LogHEAD();
718 static void LogGET(
size_t nDownloadedBytes);
720 static void LogPUT(
size_t nUploadedBytes);
722 static void LogPOST(
size_t nUploadedBytes,
723 size_t nDownloadedBytes);
725 static void LogDELETE();
729 static CPLString GetReportAsSerializedJSON();
732 struct NetworkStatisticsFileSystem
734 inline explicit NetworkStatisticsFileSystem(
const char* pszName) {
735 NetworkStatisticsLogger::EnterFileSystem(pszName);
738 inline ~NetworkStatisticsFileSystem()
740 NetworkStatisticsLogger::LeaveFileSystem();
744 struct NetworkStatisticsFile
746 inline explicit NetworkStatisticsFile(
const char* pszName) {
747 NetworkStatisticsLogger::EnterFile(pszName);
750 inline ~NetworkStatisticsFile()
752 NetworkStatisticsLogger::LeaveFile();
756 struct NetworkStatisticsAction
758 inline explicit NetworkStatisticsAction(
const char* pszName) {
759 NetworkStatisticsLogger::EnterAction(pszName);
762 inline ~NetworkStatisticsAction()
764 NetworkStatisticsLogger::LeaveAction();
769 int VSICURLGetDownloadChunkSize();
771 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
773 VSICurlReadCbkFunc pfnReadCbk,
774 void *pReadCbkUserData );
775 size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
776 size_t nmemb,
void *req );
777 void MultiPerform(CURLM* hCurlMultiHandle,
778 CURL* hEasyHandle =
nullptr);
779 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:442
Convenient string class based on std::string.
Definition: cpl_string.h:333
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:576
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1007
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1216
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
Various convenience functions for working with strings and string lists.
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:318
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:194
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140