QGIS API Documentation 3.37.0-Master (fdefdf9c27f)
qgsfileutils.cpp
Go to the documentation of this file.
1/***************************************************************************
2 qgsfileutils.cpp
3 ---------------------
4 begin : November 2017
5 copyright : (C) 2017 by Etienne Trimaille
6 email : etienne.trimaille at gmail dot com
7 ***************************************************************************
8 * *
9 * This program is free software; you can redistribute it and/or modify *
10 * it under the terms of the GNU General Public License as published by *
11 * the Free Software Foundation; either version 2 of the License, or *
12 * (at your option) any later version. *
13 * *
14 ***************************************************************************/
15#include "qgsfileutils.h"
16#include "qgis.h"
17#include "qgsexception.h"
18#include "qgsconfig.h"
19#include "qgsproviderregistry.h"
20#include "qgsprovidermetadata.h"
21
22#include <QObject>
23#include <QRegularExpression>
24#include <QFileInfo>
25#include <QDir>
26#include <QSet>
27#include <QDirIterator>
28
29#ifdef Q_OS_UNIX
30// For getrlimit()
31#include <sys/resource.h>
32#include <sys/time.h>
33#endif
34
35#ifdef _MSC_VER
36#include <Windows.h>
37#include <ShlObj.h>
38#pragma comment(lib,"Shell32.lib")
39#endif
40
41QString QgsFileUtils::representFileSize( qint64 bytes )
42{
43 QStringList list;
44 list << QObject::tr( "KB" ) << QObject::tr( "MB" ) << QObject::tr( "GB" ) << QObject::tr( "TB" );
45
46 QStringListIterator i( list );
47 QString unit = QObject::tr( "B" );
48
49 double fileSize = bytes;
50 while ( fileSize >= 1024.0 && i.hasNext() )
51 {
52 fileSize /= 1024.0;
53 unit = i.next();
54 }
55 return QStringLiteral( "%1 %2" ).arg( QString::number( fileSize, 'f', bytes >= 1048576 ? 2 : 0 ), unit );
56}
57
58QStringList QgsFileUtils::extensionsFromFilter( const QString &filter )
59{
60 const thread_local QRegularExpression rx( QStringLiteral( "\\*\\.([a-zA-Z0-9\\.]+)" ) );
61 QStringList extensions;
62 QRegularExpressionMatchIterator matches = rx.globalMatch( filter );
63
64 while ( matches.hasNext() )
65 {
66 const QRegularExpressionMatch match = matches.next();
67 if ( match.hasMatch() )
68 {
69 QStringList newExtensions = match.capturedTexts();
70 newExtensions.pop_front(); // remove whole match
71 extensions.append( newExtensions );
72 }
73 }
74 return extensions;
75}
76
77QString QgsFileUtils::wildcardsFromFilter( const QString &filter )
78{
79 const thread_local QRegularExpression globPatternsRx( QStringLiteral( ".*\\((.*?)\\)$" ) );
80 const QRegularExpressionMatch matches = globPatternsRx.match( filter );
81 if ( matches.hasMatch() )
82 return matches.captured( 1 );
83 else
84 return QString();
85}
86
87bool QgsFileUtils::fileMatchesFilter( const QString &fileName, const QString &filter )
88{
89 QFileInfo fi( fileName );
90 const QString name = fi.fileName();
91 const QStringList parts = filter.split( QStringLiteral( ";;" ) );
92 for ( const QString &part : parts )
93 {
94 const QStringList globPatterns = wildcardsFromFilter( part ).split( ' ', Qt::SkipEmptyParts );
95 for ( const QString &glob : globPatterns )
96 {
97 const QString re = QRegularExpression::wildcardToRegularExpression( glob );
98
99 const QRegularExpression globRx( re );
100 if ( globRx.match( name ).hasMatch() )
101 return true;
102 }
103 }
104 return false;
105}
106
107QString QgsFileUtils::ensureFileNameHasExtension( const QString &f, const QStringList &extensions )
108{
109 if ( extensions.empty() || f.isEmpty() )
110 return f;
111
112 QString fileName = f;
113 bool hasExt = false;
114 for ( const QString &extension : std::as_const( extensions ) )
115 {
116 const QString extWithDot = extension.startsWith( '.' ) ? extension : '.' + extension;
117 if ( fileName.endsWith( extWithDot, Qt::CaseInsensitive ) )
118 {
119 hasExt = true;
120 break;
121 }
122 }
123
124 if ( !hasExt )
125 {
126 const QString extension = extensions.at( 0 );
127 const QString extWithDot = extension.startsWith( '.' ) ? extension : '.' + extension;
128 fileName += extWithDot;
129 }
130
131 return fileName;
132}
133
134QString QgsFileUtils::addExtensionFromFilter( const QString &fileName, const QString &filter )
135{
136 const QStringList extensions = extensionsFromFilter( filter );
137 return ensureFileNameHasExtension( fileName, extensions );
138}
139
140QString QgsFileUtils::stringToSafeFilename( const QString &string )
141{
142 const thread_local QRegularExpression rx( QStringLiteral( "[/\\\\\\?%\\*\\:\\|\"<>]" ) );
143 QString s = string;
144 s.replace( rx, QStringLiteral( "_" ) );
145 return s;
146}
147
148QString QgsFileUtils::findClosestExistingPath( const QString &path )
149{
150 if ( path.isEmpty() )
151 return QString();
152
153 QDir currentPath;
154 QFileInfo fi( path );
155 if ( fi.isFile() )
156 currentPath = fi.dir();
157 else
158 currentPath = QDir( path );
159
160 QSet< QString > visited;
161 while ( !currentPath.exists() )
162 {
163 const QString parentPath = QDir::cleanPath( currentPath.absolutePath() + QStringLiteral( "/.." ) );
164 if ( visited.contains( parentPath ) )
165 return QString(); // break circular links
166
167 if ( parentPath.isEmpty() || parentPath == QLatin1String( "." ) )
168 return QString();
169 currentPath = QDir( parentPath );
170 visited << parentPath;
171 }
172
173 const QString res = QDir::cleanPath( currentPath.absolutePath() );
174
175 if ( res == QDir::currentPath() )
176 return QString(); // avoid default to binary folder if a filename alone is specified
177
178 return res == QLatin1String( "." ) ? QString() : res;
179}
180
181QStringList QgsFileUtils::findFile( const QString &file, const QString &basePath, int maxClimbs, int searchCeilling, const QString &currentDir )
182{
183 int depth = 0;
184 QString originalFolder;
185 QDir folder;
186 const QString fileName( basePath.isEmpty() ? QFileInfo( file ).fileName() : file );
187 const QString baseFolder( basePath.isEmpty() ? QFileInfo( file ).path() : basePath );
188
189 if ( QFileInfo( baseFolder ).isDir() )
190 {
191 folder = QDir( baseFolder ) ;
192 originalFolder = folder.absolutePath();
193 }
194 else // invalid folder or file path
195 {
196 folder = QDir( QFileInfo( baseFolder ).absolutePath() );
197 originalFolder = folder.absolutePath();
198 }
199
200 QStringList searchedFolder = QStringList();
201 QString existingBase;
202 QString backupDirectory = QDir::currentPath();
203 QStringList foundFiles;
204
205 if ( !currentDir.isEmpty() && backupDirectory != currentDir && QDir( currentDir ).exists() )
206 QDir::setCurrent( currentDir );
207
208 // find the nearest existing folder
209 while ( !folder.exists() && folder.absolutePath().count( '/' ) > searchCeilling )
210 {
211
212 existingBase = folder.path();
213 if ( !folder.cdUp() )
214 folder = QFileInfo( existingBase ).absoluteDir(); // using fileinfo to move up one level
215
216 depth += 1;
217
218 if ( depth > ( maxClimbs + 4 ) ) //break early when no folders can be found
219 break;
220 }
221 bool folderExists = folder.exists();
222
223 if ( depth > maxClimbs )
224 maxClimbs = depth;
225
226 if ( folder.absolutePath().count( '/' ) < searchCeilling )
227 searchCeilling = folder.absolutePath().count( '/' ) - 1;
228
229 while ( depth <= maxClimbs && folderExists && folder.absolutePath().count( '/' ) >= searchCeilling )
230 {
231
232 QDirIterator localFinder( folder.path(), QStringList() << fileName, QDir::Files, QDirIterator::NoIteratorFlags );
233 searchedFolder.append( folder.absolutePath() );
234 if ( localFinder.hasNext() )
235 {
236 foundFiles << localFinder.next();
237 return foundFiles;
238 }
239
240
241 const QFileInfoList subdirs = folder.entryInfoList( QDir::AllDirs );
242 for ( const QFileInfo &subdir : subdirs )
243 {
244 if ( ! searchedFolder.contains( subdir.absolutePath() ) )
245 {
246 QDirIterator subDirFinder( subdir.path(), QStringList() << fileName, QDir::Files, QDirIterator::Subdirectories );
247 if ( subDirFinder.hasNext() )
248 {
249 QString possibleFile = subDirFinder.next();
250 if ( !subDirFinder.hasNext() )
251 {
252 foundFiles << possibleFile;
253 return foundFiles;
254 }
255
256 foundFiles << possibleFile;
257 while ( subDirFinder.hasNext() )
258 {
259 foundFiles << subDirFinder.next();
260 }
261 return foundFiles;
262 }
263 }
264 }
265 depth += 1;
266
267 if ( depth > maxClimbs )
268 break;
269
270 folderExists = folder.cdUp();
271 }
272
273 if ( QDir::currentPath() == currentDir && currentDir != backupDirectory )
274 QDir::setCurrent( backupDirectory );
275
276 return foundFiles;
277}
278
279#ifdef _MSC_VER
280std::unique_ptr< wchar_t[] > pathToWChar( const QString &path )
281{
282 const QString nativePath = QDir::toNativeSeparators( path );
283
284 std::unique_ptr< wchar_t[] > pathArray( new wchar_t[static_cast< uint>( nativePath.length() + 1 )] );
285 nativePath.toWCharArray( pathArray.get() );
286 pathArray[static_cast< size_t >( nativePath.length() )] = 0;
287 return pathArray;
288}
289
290
291void fileAttributesOld( HANDLE handle, DWORD &fileAttributes, bool &hasFileAttributes )
292{
293 hasFileAttributes = false;
294 BY_HANDLE_FILE_INFORMATION info;
295 if ( GetFileInformationByHandle( handle, &info ) )
296 {
297 hasFileAttributes = true;
298 fileAttributes = info.dwFileAttributes;
299 }
300}
301
302// File attributes for Windows starting from version 8.
303void fileAttributesNew( HANDLE handle, DWORD &fileAttributes, bool &hasFileAttributes )
304{
305 hasFileAttributes = false;
306#if WINVER >= 0x0602
307 _FILE_BASIC_INFO infoEx;
308 if ( GetFileInformationByHandleEx(
309 handle,
310 FileBasicInfo,
311 &infoEx, sizeof( infoEx ) ) )
312 {
313 hasFileAttributes = true;
314 fileAttributes = infoEx.FileAttributes;
315 }
316 else
317 {
318 // GetFileInformationByHandleEx() is observed to fail for FAT32, QTBUG-74759
319 fileAttributesOld( handle, fileAttributes, hasFileAttributes );
320 }
321#else
322 fileAttributesOld( handle, fileAttributes, hasFileAttributes );
323#endif
324}
325
326bool pathIsLikelyCloudStorage( QString path )
327{
328 // For OneDrive detection need the attributes of a file from the path, not the directory itself.
329 // So just grab the first file in the path.
330 QDirIterator dirIt( path, QDir::Files );
331 if ( dirIt.hasNext() )
332 {
333 path = dirIt.next();
334 }
335
336 std::unique_ptr< wchar_t[] > pathArray = pathToWChar( path );
337 const HANDLE handle = CreateFileW( pathArray.get(), 0, FILE_SHARE_READ,
338 nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, nullptr );
339 if ( handle != INVALID_HANDLE_VALUE )
340 {
341 bool hasFileAttributes = false;
342 DWORD attributes = 0;
343 fileAttributesNew( handle, attributes, hasFileAttributes );
344 CloseHandle( handle );
345 if ( hasFileAttributes )
346 {
347 /* From the Win32 API documentation:
348 *
349 * FILE_ATTRIBUTE_RECALL_ON_OPEN:
350 * When this attribute is set, it means that the file or directory has no physical representation
351 * on the local system; the item is virtual. Opening the item will be more expensive than normal,
352 * e.g. it will cause at least some of it to be fetched from a remote store
353 *
354 * FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS
355 * When this attribute is set, it means that the file or directory is not fully present locally.
356 * For a file that means that not all of its data is on local storage (e.g. it may be sparse with
357 * some data still in remote storage).
358 */
359 return ( attributes & FILE_ATTRIBUTE_RECALL_ON_OPEN )
360 || ( attributes & FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS );
361 }
362 }
363 return false;
364}
365#endif
366
368{
369#ifdef _MSC_VER
370 auto pathType = [ = ]( const QString & path ) -> Qgis::DriveType
371 {
372 std::unique_ptr< wchar_t[] > pathArray = pathToWChar( path );
373 const UINT type = GetDriveTypeW( pathArray.get() );
374 switch ( type )
375 {
376 case DRIVE_UNKNOWN:
378
379 case DRIVE_NO_ROOT_DIR:
381
382 case DRIVE_REMOVABLE:
384
385 case DRIVE_FIXED:
387
388 case DRIVE_REMOTE:
390
391 case DRIVE_CDROM:
393
394 case DRIVE_RAMDISK:
396 }
397
399
400 };
401
402 const QString originalPath = QDir::cleanPath( path );
403 QString currentPath = originalPath;
404 QString prevPath;
405 while ( currentPath != prevPath )
406 {
407 if ( pathIsLikelyCloudStorage( currentPath ) )
409
410 prevPath = currentPath;
411 currentPath = QFileInfo( currentPath ).path();
412
413 const Qgis::DriveType type = pathType( currentPath );
414 if ( type != Qgis::DriveType::Unknown && type != Qgis::DriveType::Invalid )
415 return type;
416 }
418
419#else
420 ( void )path;
421 throw QgsNotSupportedException( QStringLiteral( "Determining drive type is not supported on this platform" ) );
422#endif
423}
424
425bool QgsFileUtils::pathIsSlowDevice( const QString &path )
426{
427#ifdef ENABLE_TESTS
428 if ( path.contains( QLatin1String( "fake_slow_path_for_unit_tests" ) ) )
429 return true;
430#endif
431
432 try
433 {
434 const Qgis::DriveType type = driveType( path );
435 switch ( type )
436 {
441 return false;
442
447 return true;
448 }
449 }
450 catch ( QgsNotSupportedException & )
451 {
452
453 }
454 return false;
455}
456
457QSet<QString> QgsFileUtils::sidecarFilesForPath( const QString &path )
458{
459 QSet< QString > res;
460 const QStringList providers = QgsProviderRegistry::instance()->providerList();
461 for ( const QString &provider : providers )
462 {
465 {
466 const QStringList possibleSidecars = metadata->sidecarFilesForUri( path );
467 for ( const QString &possibleSidecar : possibleSidecars )
468 {
469 if ( QFile::exists( possibleSidecar ) )
470 res.insert( possibleSidecar );
471 }
472 }
473 }
474 return res;
475}
476
477bool QgsFileUtils::renameDataset( const QString &oldPath, const QString &newPath, QString &error, Qgis::FileOperationFlags flags )
478{
479 if ( !QFile::exists( oldPath ) )
480 {
481 error = QObject::tr( "File does not exist" );
482 return false;
483 }
484
485 const QFileInfo oldPathInfo( oldPath );
486 QSet< QString > sidecars = sidecarFilesForPath( oldPath );
488 {
489 const QString qmdPath = oldPathInfo.dir().filePath( oldPathInfo.completeBaseName() + QStringLiteral( ".qmd" ) );
490 if ( QFile::exists( qmdPath ) )
491 sidecars.insert( qmdPath );
492 }
494 {
495 const QString qmlPath = oldPathInfo.dir().filePath( oldPathInfo.completeBaseName() + QStringLiteral( ".qml" ) );
496 if ( QFile::exists( qmlPath ) )
497 sidecars.insert( qmlPath );
498 }
499
500 const QFileInfo newPathInfo( newPath );
501
502 bool res = true;
503 QStringList errors;
504 errors.reserve( sidecars.size() );
505 // first check if all sidecars CAN be renamed -- we don't want to get partly through the rename and then find a clash
506 for ( const QString &sidecar : std::as_const( sidecars ) )
507 {
508 const QFileInfo sidecarInfo( sidecar );
509 const QString newSidecarName = newPathInfo.dir().filePath( newPathInfo.completeBaseName() + '.' + sidecarInfo.suffix() );
510 if ( newSidecarName != sidecar && QFile::exists( newSidecarName ) )
511 {
512 res = false;
513 errors.append( QDir::toNativeSeparators( newSidecarName ) );
514 }
515 }
516 if ( !res )
517 {
518 error = QObject::tr( "Destination files already exist %1" ).arg( errors.join( QLatin1String( ", " ) ) );
519 return false;
520 }
521
522 if ( !QFile::rename( oldPath, newPath ) )
523 {
524 error = QObject::tr( "Could not rename %1" ).arg( QDir::toNativeSeparators( oldPath ) );
525 return false;
526 }
527
528 for ( const QString &sidecar : std::as_const( sidecars ) )
529 {
530 const QFileInfo sidecarInfo( sidecar );
531 const QString newSidecarName = newPathInfo.dir().filePath( newPathInfo.completeBaseName() + '.' + sidecarInfo.suffix() );
532 if ( newSidecarName == sidecar )
533 continue;
534
535 if ( !QFile::rename( sidecar, newSidecarName ) )
536 {
537 errors.append( QDir::toNativeSeparators( sidecar ) );
538 res = false;
539 }
540 }
541 if ( !res )
542 {
543 error = QObject::tr( "Could not rename %1" ).arg( errors.join( QLatin1String( ", " ) ) );
544 }
545
546 return res;
547}
548
550{
551#ifdef Q_OS_UNIX
552 struct rlimit rescLimit;
553 if ( getrlimit( RLIMIT_NOFILE, &rescLimit ) == 0 )
554 {
555 return rescLimit.rlim_cur;
556 }
557#endif
558 return -1;
559}
560
562{
563#ifdef Q_OS_LINUX
564 int res = static_cast<int>( QDir( "/proc/self/fd" ).entryList().size() );
565 if ( res == 0 )
566 res = -1;
567 return res;
568#else
569 return -1;
570#endif
571}
572
574{
575 const int nFileLimit = QgsFileUtils::openedFileLimit();
576 const int nFileCount = QgsFileUtils::openedFileCount();
577 // We need some margin as Qt will crash if it cannot create some file descriptors
578 constexpr int SOME_MARGIN = 20;
579 return nFileCount > 0 && nFileLimit > 0 && nFileCount + filesToBeOpened > nFileLimit - SOME_MARGIN;
580}
581
582QStringList QgsFileUtils::splitPathToComponents( const QString &input )
583{
584 QStringList result;
585 QString path = QDir::cleanPath( input );
586 if ( path.isEmpty() )
587 return result;
588
589 const QString fileName = QFileInfo( path ).fileName();
590 if ( !fileName.isEmpty() )
591 result << fileName;
592 else if ( QFileInfo( path ).path() == path )
593 result << path;
594
595 QString prevPath = path;
596 while ( ( path = QFileInfo( path ).path() ).length() < prevPath.length() )
597 {
598 const QString dirName = QDir( path ).dirName();
599 if ( dirName == QLatin1String( "." ) )
600 break;
601
602 result << ( !dirName.isEmpty() ? dirName : path );
603 prevPath = path;
604 }
605
606 std::reverse( result.begin(), result.end() );
607 return result;
608}
609
610QString QgsFileUtils::uniquePath( const QString &path )
611{
612 if ( ! QFileInfo::exists( path ) )
613 {
614 return path;
615 }
616
617 QFileInfo info { path };
618 const QString suffix { info.completeSuffix() };
619 const QString pathPattern { QString( suffix.isEmpty() ? path : path.chopped( suffix.length() + 1 ) ).append( suffix.isEmpty() ? QStringLiteral( "_%1" ) : QStringLiteral( "_%1." ) ).append( suffix ) };
620 int i { 2 };
621 QString uniquePath { pathPattern.arg( i ) };
622 while ( QFileInfo::exists( uniquePath ) )
623 {
624 ++i;
625 uniquePath = pathPattern.arg( i );
626 }
627 return uniquePath;
628}
DriveType
Drive types.
Definition: qgis.h:858
@ Fixed
Fixed drive.
@ Invalid
Invalid path.
@ Unknown
Unknown type.
@ RamDisk
RAM disk.
@ Cloud
Cloud storage – files may be remote or locally stored, depending on user configuration.
@ Removable
Removable drive.
@ Remote
Remote drive.
@ IncludeMetadataFile
Indicates that any associated .qmd metadata file should be included with the operation.
@ IncludeStyleFile
Indicates that any associated .qml styling file should be included with the operation.
QFlags< FileOperationFlag > FileOperationFlags
File operation flags.
Definition: qgis.h:1809
static QString uniquePath(const QString &path)
Creates a unique file path name from a desired path by appending "_<n>" (where "<n>" is an integer nu...
static QString stringToSafeFilename(const QString &string)
Converts a string to a safe filename, replacing characters which are not safe for filenames with an '...
static QStringList findFile(const QString &file, const QString &basepath=QString(), int maxClimbs=4, int searchCeiling=4, const QString &currentDir=QString())
Will check basepath in an outward spiral up to maxClimbs levels to check if file exists.
static int openedFileCount()
Returns the number of currently opened files by the process.
static QString wildcardsFromFilter(const QString &filter)
Given a filter string like "GeoTIFF Files (*.tiff *.tif)", extracts the wildcard portion of this filt...
static bool renameDataset(const QString &oldPath, const QString &newPath, QString &error, Qgis::FileOperationFlags flags=Qgis::FileOperationFlag::IncludeMetadataFile|Qgis::FileOperationFlag::IncludeStyleFile)
Renames the dataset at oldPath to newPath, renaming both the file at oldPath and all associated sidec...
static QSet< QString > sidecarFilesForPath(const QString &path)
Returns a list of the sidecar files which exist for the dataset a the specified path.
static bool pathIsSlowDevice(const QString &path)
Returns true if the specified path is assumed to reside on a slow device, e.g.
static bool isCloseToLimitOfOpenedFiles(int filesToBeOpened=1)
Returns whether when opening new file(s) will reach, or nearly reach, the limit of simultaneously ope...
static Qgis::DriveType driveType(const QString &path)
Returns the drive type for the given path.
static bool fileMatchesFilter(const QString &fileName, const QString &filter)
Returns true if the given fileName matches a file filter string.
static QString ensureFileNameHasExtension(const QString &fileName, const QStringList &extensions)
Ensures that a fileName ends with an extension from the provided list of extensions.
static QString representFileSize(qint64 bytes)
Returns the human size from bytes.
static QString addExtensionFromFilter(const QString &fileName, const QString &filter)
Ensures that a fileName ends with an extension from the specified filter string.
static int openedFileLimit()
Returns the limit of simultaneously opened files by the process.
static QStringList splitPathToComponents(const QString &path)
Given a file path, returns a list of all the components leading to that path.
static QString findClosestExistingPath(const QString &path)
Returns the top-most existing folder from path.
static QStringList extensionsFromFilter(const QString &filter)
Returns a list of the extensions contained within a file filter string.
Custom exception class which is raised when an operation is not supported.
Definition: qgsexception.h:118
Holds data provider key, description, and associated shared library file or function pointer informat...
virtual QgsProviderMetadata::ProviderCapabilities providerCapabilities() const
Returns the provider's capabilities.
@ FileBasedUris
Indicates that the provider can utilize URIs which are based on paths to files (as opposed to databas...
virtual QStringList sidecarFilesForUri(const QString &uri) const
Given a uri, returns any sidecar files which are associated with the URI and this provider.
static QgsProviderRegistry * instance(const QString &pluginPath=QString())
Means of accessing canonical single instance.
QStringList providerList() const
Returns list of available providers by their keys.
QgsProviderMetadata * providerMetadata(const QString &providerKey) const
Returns metadata of the provider or nullptr if not found.