From 7fa031b648fedeaa35eb4da8003cd521cf65c4e3 Mon Sep 17 00:00:00 2001 From: Dirk Hohndel Date: Sat, 10 Apr 2021 18:03:08 -0700 Subject: [PATCH] cloudstorage: try to pick between multiple cloud servers The backend infrastructure will soon be able to support more than one cloud server which automagically stay in sync with each other. One critical requirement for that to work is that once a session was started with one of the servers, the complete session happens with that server - we must not switch from server to server while doing a git transaction. To make sure that's the case, we aren't trying to use DNS tricks to make this load balancing scheme work, but instead try to determine at program start which server is the best one to use. Right now this is super simplistic. Two servers, one in the US, one in Europe. By default we use the European server (most of our users appear to be in Europe), but if we can figure out that the client is actually in the Americas, use the US server. We might improve that heuristic over time, but as a first attempt it seems not entirely bogus. The way this is implemented is a simple combination of two free webservices that together appear to give us a very reliable estimate which continent the user is located on. api.ipify.org gives us our external IP address ip-api.com gives us the continent that IP address is on If any of this fails or takes too long to respond, we simply ignore it since either server will work. One oddity is that if we decide to change servers we only change the settings that are stored on disk, not the runtime preferences. This goes back to the comment above that we have to avoid changing servers in mid sync. Signed-off-by: Dirk Hohndel --- core/checkcloudconnection.cpp | 77 +++++++++++++++++++++++++++++ core/checkcloudconnection.h | 3 ++ core/git-access.c | 10 +++- core/git-access.h | 5 ++ core/pref.c | 3 +- core/settings/qPrefCloudStorage.cpp | 7 ++- desktop-widgets/mainwindow.cpp | 2 +- subsurface-desktop-main.cpp | 3 ++ subsurface-mobile-main.cpp | 3 ++ 9 files changed, 105 insertions(+), 8 deletions(-) diff --git a/core/checkcloudconnection.cpp b/core/checkcloudconnection.cpp index b9a7d3b0e..714b62836 100644 --- a/core/checkcloudconnection.cpp +++ b/core/checkcloudconnection.cpp @@ -4,11 +4,14 @@ #include #include #include +#include #include "pref.h" #include "qthelper.h" #include "git-access.h" #include "errorhelper.h" +#include "core/subsurface-string.h" +#include "core/settings/qPrefCloudStorage.h" #include "checkcloudconnection.h" @@ -19,6 +22,10 @@ CheckCloudConnection::CheckCloudConnection(QObject *parent) : } +// two free APIs to figure out where we are +#define GET_EXTERNAL_IP_API "http://api.ipify.org" +#define GET_CONTINENT_API "http://ip-api.com/line/%1?fields=continent" + // our own madeup API to make sure we are talking to a Subsurface cloud server #define TEAPOT "/make-latte?number-of-shots=3" #define HTTP_I_AM_A_TEAPOT 418 @@ -84,6 +91,76 @@ void CheckCloudConnection::sslErrors(const QList &errorList) qDebug() << err.errorString(); } +void CheckCloudConnection::pickServer() +{ + QNetworkRequest request(QString(GET_EXTERNAL_IP_API)); + request.setRawHeader("Accept", "text/plain"); + request.setRawHeader("User-Agent", getUserAgent().toUtf8()); + QNetworkAccessManager *mgr = new QNetworkAccessManager(); + connect(mgr, &QNetworkAccessManager::finished, this, &CheckCloudConnection::gotIP); + mgr->get(request); +} + +void CheckCloudConnection::gotIP(QNetworkReply *reply) +{ + if (reply->error() != QNetworkReply::NoError) { + // whatever, just use the default host + if (verbose) + qDebug() << __FUNCTION__ << "got error reply from ip webservice - not changing cloud host"; + return; + } + QString addressString = reply->readAll(); + // use the QHostAddress constructor as a convenient way to validate that this is indeed an IP address + // but then don't do annything with the QHostAdress - we need the address string... + QHostAddress addr(addressString); + if (addr.isNull()) { + // this isn't an address, don't try to update the cloud host + if (verbose) + qDebug() << __FUNCTION__ << "returned address doesn't appear to be valid (" << addressString << ") - not changing cloud host"; + return; + } + if (verbose) + qDebug() << "IP used for cloud server access" << addressString; + // now figure out which continent we are on + QNetworkRequest request(QString(GET_CONTINENT_API).arg(addressString)); + request.setRawHeader("Accept", "text/plain"); + request.setRawHeader("User-Agent", getUserAgent().toUtf8()); + QNetworkAccessManager *mgr = new QNetworkAccessManager(); + connect(mgr, &QNetworkAccessManager::finished, this, &CheckCloudConnection::gotContinent); + mgr->get(request); +} + +void CheckCloudConnection::gotContinent(QNetworkReply *reply) +{ + if (reply->error() != QNetworkReply::NoError) { + // whatever, just use the default host + if (verbose) + qDebug() << __FUNCTION__ << "got error reply from ip location webservice - not changing cloud host"; + return; + } + QString continentString = reply->readAll(); + // in most cases this response comes back too late for us - we may already have + // started to talk to the cloud server (this certinaly seems to be the case when + // we use the cloud storage as default file). So instead of potentially changing + // the server that is used in mid connection, let's just update what's stored in + // our settings so the next time we'll use the server that's closer. + + // of course, right now the logic for that is very simplistic. Use the US server + // when in the Americas, the EU server otherwise. This may need a better algorithm + // at some point, but for now it seems good enough + + const char *base_url; + if (continentString.contains("America", Qt::CaseInsensitive)) + base_url = "https://" CLOUD_HOST_US "/"; + else + base_url = "https://" CLOUD_HOST_EU "/"; + if (!same_string(base_url, prefs.cloud_base_url)) { + if (verbose) + qDebug() << "remember cloud server" << base_url << "based on IP location in " << continentString; + qPrefCloudStorage::instance()->store_cloud_base_url(base_url); + } +} + // helper to be used from C code extern "C" bool canReachCloudServer() { diff --git a/core/checkcloudconnection.h b/core/checkcloudconnection.h index 312a1e78c..414ddc434 100644 --- a/core/checkcloudconnection.h +++ b/core/checkcloudconnection.h @@ -11,11 +11,14 @@ class CheckCloudConnection : public QObject { public: CheckCloudConnection(QObject *parent = 0); bool checkServer(); + void pickServer(); private: QNetworkReply *reply; private slots: void sslErrors(const QList &errorList); + void gotIP(QNetworkReply *reply); + void gotContinent(QNetworkReply *reply); }; #endif // CHECKCLOUDCONNECTION_H diff --git a/core/git-access.c b/core/git-access.c index 0936fa399..bd958b517 100644 --- a/core/git-access.c +++ b/core/git-access.c @@ -297,7 +297,8 @@ int certificate_check_cb(git_cert *cert, int valid, const char *host, void *payl UNUSED(payload); if (verbose) SSRF_INFO("git storage: certificate callback for host %s with validity %d\n", host, valid); - if (same_string(host, "cloud.subsurface-divelog.org") && cert->cert_type == GIT_CERT_X509) { + if ((same_string(host, CLOUD_HOST_GENERIC) || same_string(host, CLOUD_HOST_US) || same_string(host, CLOUD_HOST_EU)) && + cert->cert_type == GIT_CERT_X509) { // for some reason the LetsEncrypt certificate makes libgit2 throw up on some // platforms but not on others // if we are connecting to the cloud server we alrady called 'canReachCloudServer()' @@ -712,7 +713,7 @@ int sync_with_remote(git_repository *repo, const char *remote, const char *branc return 0; } if (verbose) - SSRF_INFO("git storage: fetch remote\n"); + SSRF_INFO("git storage: fetch remote %s\n", git_remote_url(origin)); git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; opts.callbacks.transfer_progress = &transfer_progress_cb; auth_attempt = 0; @@ -775,6 +776,11 @@ static git_repository *update_local_repo(const char *localdir, const char *remot } git_reference_free(head); } + /* make sure we have the correct origin - the cloud server URL could have changed */ + if (git_remote_set_url(repo, "origin", remote)) { + SSRF_INFO("git storage: failed to update origin to '%s'", remote); + return NULL; + } if (!git_local_only) sync_with_remote(repo, remote, branch, rt); diff --git a/core/git-access.h b/core/git-access.h index 7bdbff971..210ebb2fd 100644 --- a/core/git-access.h +++ b/core/git-access.h @@ -15,6 +15,11 @@ extern "C" { #include #endif +#define CLOUD_HOST_US "ssrf-cloud-us.subsurface-divelog.org" +#define CLOUD_HOST_EU "ssrf-cloud-eu.subsurface-divelog.org" +#define CLOUD_HOST_PATTERN "ssrf-cloud-..\\.subsurface-divelog\\.org" +#define CLOUD_HOST_GENERIC "cloud.subsurface-divelog.org" + enum remote_transport { RT_OTHER, RT_HTTPS, RT_SSH }; struct git_oid; diff --git a/core/pref.c b/core/pref.c index 61619c0d1..6be6fe6ee 100644 --- a/core/pref.c +++ b/core/pref.c @@ -1,10 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 #include "pref.h" #include "subsurface-string.h" +#include "git-access.h" // for CLOUD_HOST struct preferences prefs, git_prefs; struct preferences default_prefs = { - .cloud_base_url = "https://cloud.subsurface-divelog.org/", + .cloud_base_url = "https://" CLOUD_HOST_EU "/", // if we don't know any better, use the European host .units = SI_UNITS, .unit_system = METRIC, .coordinates_traditional = true, diff --git a/core/settings/qPrefCloudStorage.cpp b/core/settings/qPrefCloudStorage.cpp index 2c320400b..d08b80ce3 100644 --- a/core/settings/qPrefCloudStorage.cpp +++ b/core/settings/qPrefCloudStorage.cpp @@ -46,11 +46,10 @@ void qPrefCloudStorage::store_cloud_base_url(const QString &value) } void qPrefCloudStorage::disk_cloud_base_url(bool doSync) { - if (doSync) { - qPrefPrivate::propSetValue(keyFromGroupAndName(group, "cloud_base_url"), prefs.cloud_base_url, default_prefs.cloud_base_url); - } else { + // we don't allow to automatically write back the prefs value for the cloud_base_url. + // in order to do that you need to use the explicit function above store_cloud_base_url() + if (!doSync) prefs.cloud_base_url = copy_qstring(qPrefPrivate::propValue(keyFromGroupAndName(group, "cloud_base_url"), default_prefs.cloud_base_url).toString()); - } } HANDLE_PREFERENCE_TXT(CloudStorage, "email", cloud_storage_email); diff --git a/desktop-widgets/mainwindow.cpp b/desktop-widgets/mainwindow.cpp index 9a2b9665d..7ea150dd1 100644 --- a/desktop-widgets/mainwindow.cpp +++ b/desktop-widgets/mainwindow.cpp @@ -1262,7 +1262,7 @@ int MainWindow::file_save_as(void) // if the default is to save to cloud storage, pick something that will work as local file: // simply extract the branch name which should be the users email address - if (default_filename && strstr(default_filename, prefs.cloud_base_url)) { + if (default_filename && QString(default_filename).contains(QRegularExpression(CLOUD_HOST_PATTERN))) { QString filename(default_filename); filename.remove(0, filename.indexOf("[") + 1); filename.replace("]", ".ssrf"); diff --git a/subsurface-desktop-main.cpp b/subsurface-desktop-main.cpp index f42361538..bb203e21c 100644 --- a/subsurface-desktop-main.cpp +++ b/subsurface-desktop-main.cpp @@ -15,6 +15,7 @@ #include "core/settings/qPref.h" #include "core/tag.h" #include "desktop-widgets/mainwindow.h" +#include "core/checkcloudconnection.h" #include #include @@ -76,6 +77,8 @@ int main(int argc, char **argv) #endif setup_system_prefs(); copy_prefs(&default_prefs, &prefs); + CheckCloudConnection ccc; + ccc.pickServer(); fill_computer_list(); reset_tank_info_table(&tank_info_table); parse_xml_init(); diff --git a/subsurface-mobile-main.cpp b/subsurface-mobile-main.cpp index 454996fc7..0020116da 100644 --- a/subsurface-mobile-main.cpp +++ b/subsurface-mobile-main.cpp @@ -17,6 +17,7 @@ #include "core/settings/qPrefDisplay.h" #include "core/tag.h" #include "core/settings/qPrefCloudStorage.h" +#include "core/checkcloudconnection.h" #include #include @@ -57,6 +58,8 @@ int main(int argc, char **argv) else default_prefs.units = IMPERIAL_units; copy_prefs(&default_prefs, &prefs); + CheckCloudConnection ccc; + ccc.pickServer(); fill_computer_list(); reset_tank_info_table(&tank_info_table);