Skip to content

Commit

Permalink
QF-1414 - Write data as it is being made available when downloading Q…
Browse files Browse the repository at this point in the history 
…FieldCloud projects to avoid running out of memory (#5905)

* Download and writing data to a temporary file in QFieldCloudProjects.

Incrementally saving downloaded data to a temporary file during each progress update.

* Ui polish.

1- Allow changing tabs ["My Projects" / "Community"].
2- Fix invisible progressBar under headersection (layer name).
3- Fix spacing between progressBar and next item.

* Bring back `QgsMessageLog::logMessage` and update docs.

* Update NetworkReply `*reply()`s documentation.

* Rename reply to currentRawReply and add currentRawReplyChanged.

* Remove `projectId` from `logFailedDownload` and fix its signature.

* Fix compilation following merging of the background positioning PR

---------

Co-authored-by: Mathieu Pellerin <[email protected]>
  • Loading branch information
@mohsenD98 @nirvn
mohsenD98 and nirvn authored Dec 30, 2024
1 parent 043f4cc commit 23ca5bc
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 72 deletions.
8 changes: 3 additions & 5 deletions src/core/networkreply.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,9 @@ void NetworkReply::abort()
}


QNetworkReply *NetworkReply::reply() const
QNetworkReply *NetworkReply::currentRawReply() const
{
if ( mIsFinished )
return mReply;

return nullptr;
return mReply;
}


Expand Down Expand Up @@ -90,6 +87,7 @@ void NetworkReply::initiateRequest()
case QNetworkAccessManager::UnknownOperation:
throw QStringLiteral( "Not implemented!" );
}
emit currentRawReplyChanged();

mReply->ignoreSslErrors( mExpectedSslErrors );

Expand Down
11 changes: 8 additions & 3 deletions src/core/networkreply.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,10 @@ class NetworkReply : public QObject


/**
* Get the QNetworkReply object once the CloudReply is finilized. Do not delete it manually.
* @return network reply
* Get the current `QNetworkReply` object. Note that it might get deleted even if the parent `NetworkReply` is not in case of redirect or internal retry. Do not delete it manually.
* @return network currentRawReply
*/
QNetworkReply *reply() const;
QNetworkReply *currentRawReply() const;


/**
Expand Down Expand Up @@ -145,6 +145,11 @@ class NetworkReply : public QObject
*/
void temporaryErrorOccurred( QNetworkReply::NetworkError code );

/**
* Emitted when reply has changed.
*/
void currentRawReplyChanged();

private:
/**
* Upper bound of the delay between retries in milliseconds.
Expand Down
10 changes: 5 additions & 5 deletions src/core/qfieldcloudconnection.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ void QFieldCloudConnection::login()

// Handle login redirect as an error state
connect( reply, &NetworkReply::redirected, this, [=]() {
QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();
reply->deleteLater();
rawReply->deleteLater();

Expand All @@ -175,7 +175,7 @@ void QFieldCloudConnection::login()
} );

connect( reply, &NetworkReply::finished, this, [=]() {
QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();

Q_ASSERT( reply->isFinished() );
Q_ASSERT( rawReply );
Expand Down Expand Up @@ -334,7 +334,7 @@ NetworkReply *QFieldCloudConnection::post( const QString &endpoint, const QVaria
mPendingRequests++;
setState( ConnectionState::Busy );
connect( reply, &NetworkReply::finished, this, [=]() {
QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();
if ( --mPendingRequests == 0 )
{
if ( rawReply->error() != QNetworkReply::NoError )
Expand Down Expand Up @@ -395,7 +395,7 @@ NetworkReply *QFieldCloudConnection::get( QNetworkRequest &request, const QUrl &
mPendingRequests++;
setState( ConnectionState::Busy );
connect( reply, &NetworkReply::finished, this, [=]() {
QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();
if ( --mPendingRequests == 0 )
{
if ( rawReply->error() != QNetworkReply::NoError )
Expand Down Expand Up @@ -602,7 +602,7 @@ void QFieldCloudConnection::processPendingAttachments()
QString projectId = it.key();
QString fileName = it.value();
connect( attachmentCloudReply, &NetworkReply::finished, this, [=]() {
QNetworkReply *attachmentReply = attachmentCloudReply->reply();
QNetworkReply *attachmentReply = attachmentCloudReply->currentRawReply();
attachmentCloudReply->deleteLater();

Q_ASSERT( attachmentCloudReply->isFinished() );
Expand Down
114 changes: 68 additions & 46 deletions src/core/qfieldcloudprojectsmodel.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@ void QFieldCloudProjectsModel::refreshProjectFileOutdatedStatus( const QString &
return;
}

QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();
reply->deleteLater();

if ( rawReply->error() != QNetworkReply::NoError )
Expand Down Expand Up @@ -519,7 +519,7 @@ void QFieldCloudProjectsModel::projectRefreshData( const QString &projectId, con
if ( !findProject( projectId ) )
return;

QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();

reply->deleteLater();

Expand Down Expand Up @@ -620,7 +620,7 @@ void QFieldCloudProjectsModel::projectStartJob( const QString &projectId, const
return;
}

QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();

if ( rawReply->error() != QNetworkReply::NoError )
{
Expand Down Expand Up @@ -699,7 +699,7 @@ void QFieldCloudProjectsModel::projectGetJobStatus( const QString &projectId, co
return;
}

QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();

if ( rawReply->error() != QNetworkReply::NoError )
{
Expand Down Expand Up @@ -1005,7 +1005,7 @@ void QFieldCloudProjectsModel::projectDownload( const QString &projectId )
return;
}

QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();

reply->deleteLater();

Expand Down Expand Up @@ -1379,7 +1379,7 @@ void QFieldCloudProjectsModel::projectUpload( const QString &projectId, const bo
emit dataChanged( projectIndex, projectIndex, QVector<int>() << UploadDeltaProgressRole );
} );
connect( deltasCloudReply, &NetworkReply::finished, this, [=]() {
QNetworkReply *deltasReply = deltasCloudReply->reply();
QNetworkReply *deltasReply = deltasCloudReply->currentRawReply();
deltasCloudReply->deleteLater();

Q_ASSERT( deltasCloudReply->isFinished() );
Expand Down Expand Up @@ -1561,7 +1561,7 @@ void QFieldCloudProjectsModel::refreshProjectDeltaList( const QString &projectId
}

connect( deltaStatusReply, &NetworkReply::finished, this, [=]() {
QNetworkReply *rawReply = deltaStatusReply->reply();
QNetworkReply *rawReply = deltaStatusReply->currentRawReply();
deltaStatusReply->deleteLater();

Q_ASSERT( deltaStatusReply->isFinished() );
Expand Down Expand Up @@ -1591,7 +1591,7 @@ void QFieldCloudProjectsModel::projectGetDeltaStatus( const QString &projectId )

project->deltaFileUploadStatusString = QString();
connect( deltaStatusReply, &NetworkReply::finished, this, [=]() {
QNetworkReply *rawReply = deltaStatusReply->reply();
QNetworkReply *rawReply = deltaStatusReply->currentRawReply();
deltaStatusReply->deleteLater();

Q_ASSERT( deltaStatusReply->isFinished() );
Expand Down Expand Up @@ -1688,7 +1688,7 @@ void QFieldCloudProjectsModel::layerObserverLayerEdited( const QString &layerId
void QFieldCloudProjectsModel::projectListReceived()
{
NetworkReply *reply = qobject_cast<NetworkReply *>( sender() );
QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();

Q_ASSERT( rawReply );

Expand Down Expand Up @@ -1736,7 +1736,6 @@ NetworkReply *QFieldCloudProjectsModel::downloadFile( const QString &projectId,
return mCloudConnection->get( request, QStringLiteral( "/api/v1/packages/%1/latest/files/%2/" ).arg( projectId, fileName ) );
}


void QFieldCloudProjectsModel::downloadFileConnections( const QString &projectId, const QString &fileName )
{
const QModelIndex projectIndex = findProjectIndex( projectId );
Expand Down Expand Up @@ -1809,6 +1808,44 @@ void QFieldCloudProjectsModel::downloadFileConnections( const QString &projectId
} );

connect( reply, &NetworkReply::downloadProgress, reply, [=]( int bytesReceived, int bytesTotal ) {
QNetworkReply *rawReply = reply->currentRawReply();
if ( !rawReply )
{
return;
}

const QString temporaryFileName = project->downloadFileTransfers[fileName].tmpFile;
QFile file( temporaryFileName );
QString errorMessageDetail;
QString errorMessage;
bool hasError = false;

if ( file.open( QIODevice::WriteOnly | QIODevice::Append ) )
{
file.write( rawReply->readAll() );

if ( file.error() != QFile::NoError )
{
hasError = true;
errorMessageDetail = file.errorString();
errorMessage = tr( "File system error. Failed to write file to temporary location `%1`." ).arg( temporaryFileName );
}
}
else
{
hasError = true;
errorMessageDetail = file.errorString();
errorMessage = tr( "File system error. Failed to open file for writing on temporary `%1`." ).arg( temporaryFileName );
}

// check if the code above failed with error
if ( hasError )
{
logFailedDownload( project, fileName, errorMessage, errorMessageDetail );
rawReply->abort();
return;
}

if ( !findProject( projectId ) )
{
QgsLogger::debug( QStringLiteral( "Project %1, file `%2`: updating download progress, but the project is deleted." ).arg( projectId, fileName ) );
Expand All @@ -1835,7 +1872,7 @@ void QFieldCloudProjectsModel::downloadFileConnections( const QString &projectId
}

QVector<int> rolesChanged;
QNetworkReply *rawReply = reply->reply();
QNetworkReply *rawReply = reply->currentRawReply();

Q_ASSERT( reply->isFinished() );
Q_ASSERT( reply );
Expand Down Expand Up @@ -1863,46 +1900,13 @@ void QFieldCloudProjectsModel::downloadFileConnections( const QString &projectId
project->downloadBytesReceived += project->downloadFileTransfers[fileName].bytesTotal;
project->downloadProgress = std::clamp( ( static_cast<double>( project->downloadBytesReceived ) / std::max( project->downloadBytesTotal, 1 ) ), 0., 1. );
emit dataChanged( projectIndex, projectIndex, QVector<int>() << DownloadProgressRole );

QFile file( project->downloadFileTransfers[fileName].tmpFile );

if ( file.open( QIODevice::ReadWrite ) )
{
file.write( rawReply->readAll() );

if ( file.error() != QFile::NoError )
{
hasError = true;
errorMessageDetail = file.errorString();
errorMessage = tr( "File system error. Failed to write file to temporary location `%1`." ).arg( project->downloadFileTransfers[fileName].tmpFile );
}
}
else
{
hasError = true;
errorMessageDetail = file.errorString();
errorMessage = tr( "File system error. Failed to open file for writing on temporary `%1`." ).arg( project->downloadFileTransfers[fileName].tmpFile );
}
}

// check if the code above failed with error
if ( hasError )
{
project->downloadFilesFailed++;

QgsLogger::debug( QStringLiteral( "Project %1, file `%2`: %3 %4" ).arg( errorMessage, fileName, errorMessage, errorMessageDetail ) );

// translate the user messages
const QString baseMessage = tr( "Project `%1`, file `%2`: %3" ).arg( project->name, fileName, errorMessage );
const QString trimmedMessage = baseMessage + QStringLiteral( " " ) + tr( "System message: " )
+ ( ( errorMessageDetail.size() > 100 )
? ( errorMessageDetail.left( 100 ) + tr( " (see more in the QField error log)…" ) )
: errorMessageDetail );

QgsMessageLog::logMessage( QStringLiteral( "%1\n%2" ).arg( baseMessage, errorMessageDetail ) );

emit projectDownloadFinished( projectId, trimmedMessage );

logFailedDownload( project, fileName, errorMessage, errorMessageDetail );
rawReply->abort();
return;
}

Expand Down Expand Up @@ -2080,6 +2084,24 @@ void QFieldCloudProjectsModel::insertProjects( const QList<CloudProject *> &proj
endInsertRows();
}

void QFieldCloudProjectsModel::logFailedDownload( CloudProject *project, const QString &fileName, const QString &errorMessage, const QString &errorMessageDetail )
{
project->downloadFilesFailed++;

QgsLogger::debug( QStringLiteral( "Project %1, file `%2`: %3 %4" ).arg( errorMessage, fileName, errorMessage, errorMessageDetail ) );

// translate the user messages
const QString baseMessage = tr( "Project `%1`, file `%2`: %3" ).arg( project->name, fileName, errorMessage );
const QString trimmedMessage = baseMessage + QStringLiteral( " " ) + tr( "System message: " )
+ ( ( errorMessageDetail.size() > 100 )
? ( errorMessageDetail.left( 100 ) + tr( " (see more in the QField error log)…" ) )
: errorMessageDetail );

QgsMessageLog::logMessage( QStringLiteral( "%1\n%2" ).arg( baseMessage, errorMessageDetail ) );

emit projectDownloadFinished( project->id, trimmedMessage );
}

void QFieldCloudProjectsModel::loadProjects( const QJsonArray &remoteProjects, bool skipLocalProjects )
{
QgsProject *qgisProject = QgsProject::instance();
Expand Down
1 change: 1 addition & 0 deletions src/core/qfieldcloudprojectsmodel.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,7 @@ class QFieldCloudProjectsModel : public QAbstractListModel
void downloadFileConnections( const QString &projectId, const QString &fileName );
void loadProjects( const QJsonArray &remoteProjects = QJsonArray(), bool skipLocalProjects = false );
void insertProjects( const QList<CloudProject *> &projects );
void logFailedDownload( CloudProject *project, const QString &fileName, const QString &errorMessage, const QString &errorMessageDetail );
};

Q_DECLARE_METATYPE( QFieldCloudProjectsModel::ProjectStatus )
Expand Down
31 changes: 18 additions & 13 deletions src/qml/QFieldCloudScreen.qml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,6 @@ Page {
height: filterBar.defaultHeight
width: projects.width / filterBar.count
font: Theme.defaultFont
enabled: (cloudConnection.state === QFieldCloudConnection.Idle && cloudProjectsModel.busyProjectIds.length === 0)
onClicked: {
filterBar.currentIndex = index;
}
Expand Down Expand Up @@ -224,20 +223,26 @@ Page {
section.labelPositioning: ViewSection.CurrentLabelAtStart | ViewSection.InlineLabels
section.delegate: Component {
/* section header: layer name */
Rectangle {
Item {
width: parent.width
height: 30
color: Theme.controlBorderColor
height: 32

Text {
anchors {
horizontalCenter: parent.horizontalCenter
verticalCenter: parent.verticalCenter
Rectangle {
width: parent.width
height: 30
color: Theme.controlBorderColor
anchors.bottom: parent.bottom

Text {
anchors {
horizontalCenter: parent.horizontalCenter
verticalCenter: parent.verticalCenter
}
font.bold: true
font.pointSize: Theme.resultFont.pointSize
color: Theme.mainTextColor
text: section
}
font.bold: true
font.pointSize: Theme.resultFont.pointSize
color: Theme.mainTextColor
text: section
}
}
}
Expand Down Expand Up @@ -266,7 +271,7 @@ Page {
property int status: Status

width: parent ? parent.width : undefined
height: line.height
height: line.height + 6
color: "transparent"

ProgressBar {
Expand Down

1 comment on commit 23ca5bc

@qfield-fairy
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎉 Ta-daaa, freshly created APKs are available: arm64-android

Other architectures

Please sign in to comment.