Gpt4all Custom Updater

This PR establishes the initial infrastructure required to migrate gpt4all away from reliance on the IFW framwork by implementing a custom updater.
This custom updater (currently headless only) can perform the same operations as the existing updater with the option for more functionality to be added and most importantly, is installer agnostic, meaning gpt4all can start to leverage platform specific installers.

Implements both offline and online installation and update mechanisms.

Initial implementation of: #2878

Signed-off-by: John Parent <[email protected]>

Author: johnwparent

gpt4all-updater/CMakeLists.txt

@@ -0,0 +1,73 @@
+cmake_minimum_required(VERSION 3.16)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+set(APP_VERSION_MAJOR 0)
+set(APP_VERSION_MINOR 0)
+set(APP_VERSION_PATCH  0)
+
+set(APP_VERSION ${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH})
+
+project(Gpt4AllAutoUpdater VERSION ${APP_VERSION} LANGUAGES CXX)
+
+option(BUILD_OFFLINE_UPDATER "Build an offline updater" OFF)
+
+if(BUILD_OFFLINE_UPDATER AND NOT GPT4ALL_INSTALLER_PATH)
+    message(FATAL_ERROR "The path to GPT4ALL's installer is required to construct this updater.
+Please provide it on the command line using the argument -DGPT4ALL_INSTALLER_PATH=<pth>")
+endif()
+
+if(NOT BUILD_OFFLINE_UPDATER AND NOT GPT4ALL_MANIFEST_ENDPOINT)
+    message(FATAL_ERROR "The manifest endpoint was not provided, the online installer will be unable to detect updates")
+endif()
+
+if(APPLE)
+    option(BUILD_UNIVERSAL "Build universal binary on MacOS" OFF)
+    if(BUILD_UNIVERSAL)
+        set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64" CACHE STRING "" FORCE)
+    else()
+        set(CMAKE_OSX_ARCHITECTURES "${CMAKE_HOST_SYSTEM_PROCESSOR}" CACHE STRING "" FORCE)
+    endif()
+endif()
+
+if(APPLE AND BUILD_OFFLINE_UPDATER)
+    enable_language(ASM)
+    configure_file(src/asm/bake_installer.S.in ${CMAKE_BINARY_DIR}/bake_installer.S @ONLY)
+    set(ASSEMBLER_SOURCES ${CMAKE_BINARY_DIR}/bake_installer.S src/Embedded.cxx)
+elseif(WIN32 AND BUILD_OFFLINE_UPDATER)
+    configure_file(src/resources/installer.rc.in ${CMAKE_BINARY_DIR}/resource.rc @ONLY)
+    set(RC_FILES ${CMAKE_BINARY_DIR}/resource.rc src/Resource.cxx)
+endif()
+
+if(NOT BUILD_OFFLINE_UPDATER)
+    configure_file(src/Download.cxx.in ${CMAKE_BINARY_DIR}/Download.cxx @ONLY)
+endif()
+
+find_package(Qt6 REQUIRED COMPONENTS Core Network)
+
+set(auto_updater_sources
+    src/Command.cxx
+    src/CommandFactory.cxx
+    src/CommandLine.cxx
+    src/Downgrade.cxx
+    ${CMAKE_BINARY_DIR}/Download.cxx
+    src/Manifest.cxx
+    src/Modify.cxx
+    src/Package.cxx
+    src/State.cxx
+    src/Uninstall.cxx
+    src/Update.cxx
+    src/utils.cxx
+    src/main.cxx
+    ${ASSEMBLER_SOURCES}
+    ${RC_FILES}
+)
+
+add_executable(autoupdater ${auto_updater_sources})
+target_link_libraries(autoupdater PRIVATE Qt6::Core Qt6::Network)
+target_include_directories(autoupdater PRIVATE include)
+
+if(BUILD_OFFLINE_UPDATER)
+    target_compile_definitions(autoupdater PRIVATE OFFLINE)
+endif()

gpt4all-updater/README

@@ -0,0 +1,30 @@
+# Gpt4All Updater
+
+## Testing
+
+Testing this updater requires a bit of manual setup and walking through the
+integration with gpt4all's installer, as this process has yet to be fully automated.
+
+There are two modes, offline and online, each is tested differently:
+
+### Online
+
+The online updater workflow takes a url endpoint (for early testing, this will be a file url) that points to a manifest file, a sample of which has been provided under the `tmp` directory relative to this README. The manifest file includes, amount other things specified in the updater RFC, another file URL to a gpt4all installer.
+
+The first thing testing this will require is a manual update of that file url in the manifest xml file to point to the DMG on MacOS or exe on Windows, of an existing online Gpt4All installer, as well as a corresponding sha256 sum of the given installer. The manifest is filled with other stub info for testing, you're welcome to leave it as is or fill it out correctly for each testing iteration.
+
+That is all that is required for configuration. Now simply build this project via CMake, using standard CMake build practices. CMake will build the online installer by default, so no extra arguments are required.
+
+One argument is required for the online installer, the url where the updater should expect the manifest file to be. This can be any url accessible to your system, but for simplicity and testing reasons, its usually best to use a file url.
+This is provided via the cmake command line argument `-DGPT4ALL_MANIFEST_ENDPOINT=<url>`.
+
+Now configure and build the updater with CMake.
+
+To test the installer, query the command line interface using the `--help` argument to determine which actions are available. Then select a given action, and provide the required arguments (there shouldn't be any at the moment), and let the updater drive. The updater will determine the operation requested, fetch the appropriate installer, and drive said installer with the appropriate arguments to effect the requested operation. If the operation is a modification or uninstall, the updater will not fetch a new installer, and instead will execute the older installer, as a new installer is not required.
+
+### Offline
+
+The offline updater is somewhat simpler. To instruct CMake to build the offline updater, specify the `-DBUILD_OFFLINE_UPDATER=ON` argument to CMake on the command line. One additional argument is required to properly configure CMake for the offline updater project, `-DGPT4ALL_INSTALLER_PATH`  which should be set to the path to the DMG file containing an offline version of the GPT4All installer.
+
+Now, after building, simply run the updater. It should remove your current installation of Gpt4All (but not the config or models), and then run the offline installer in install mode. Once that process is complete, you should have an upgraded Gpt4All available on your system.
+

gpt4all-updater/include/Command.h

@@ -0,0 +1,18 @@
+#pragma once
+#include <QCoreApplication>
+#include <QProcess>
+#include <QStringList>
+#include <QFile>
+
+namespace gpt4all {
+namespace command{
+
+class Command
+{
+public:
+    virtual bool execute();
+    QStringList arguments;
+    QFile* installer;
+};
+}
+}

gpt4all-updater/include/CommandFactory.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "Command.h"
+#include "CommandLine.h"
+
+
+
+class CommandFactory : public QObject
+{
+public:
+    std::shared_ptr<gpt4all::command::Command> GetCommand(gpt4all::command::CommandType type);
+};

gpt4all-updater/include/CommandLine.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <QCommandLineParser>
+
+
+namespace gpt4all {
+namespace command {
+    
+enum CommandType {
+    UPDATE,
+    MODIFY,
+    DOWNGRADE,
+    UNINSTALL
+};
+
+class CommandLine : public QObject
+{
+public:
+    CommandLine();
+    ~CommandLine();
+    void parse(QCoreApplication &app);
+    CommandType command();
+private:
+    CommandType type;
+    QCommandLineParser * parser;
+};
+
+}
+}
+

gpt4all-updater/include/Downgrade.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "Command.h"
+
+
+namespace gpt4all {
+namespace downgrade {
+
+class Downgrade : public gpt4all::command::Command
+{
+public:
+    Downgrade(QFile &installer);
+};
+
+
+}
+}

gpt4all-updater/include/Download.h

@@ -0,0 +1,65 @@
+#pragma once
+
+#include "Manifest.h"
+#include "State.h"
+
+#include <QCryptographicHash>
+#include <QDateTime>
+#include <QFile>
+#include <QHash>
+#include <QList>
+#include <QMap>
+#include <QObject>
+#include <QtGlobal>
+#include <QVersionNumber>
+#include <QSslError>
+#include <QThread>
+#include <QString>
+#include <QNetworkAccessManager>
+#include <QNetworkReply>
+#include <QCoreApplication>
+
+namespace gpt4all {
+namespace download {
+
+class HashFile : public QObject
+{
+public:
+    HashFile() : QObject(nullptr) {}
+    void hashAndInstall(const QString &expected, QCryptographicHash::Algorithm algo, QFile *temp, const QString &file, QNetworkReply *reply);
+};
+
+class Download : public QObject
+{
+public:
+    static Download *instance();
+    Q_INVOKABLE void driveFetchAndInstall();
+    Q_INVOKABLE void downloadManifest();
+    Q_INVOKABLE void downloadInstaller();
+    Q_INVOKABLE void cancelDownload();
+    Q_INVOKABLE void installInstaller(QString &expected, QFile *temp, QNetworkReply *installerResponse);
+
+private Q_SLOTS:
+    void handleSslErrors(QNetworkReply *reply, const QList<QSslError> &errors);
+    void handleErrorOccurred(QNetworkReply::NetworkError code);
+    void handleInstallerDownloadFinished();
+    void handleReadyRead();
+
+private:
+    explicit Download();
+    ~Download();
+    QNetworkReply * downloadInMemory(QUrl &url);
+    QNetworkReply * downloadLargeFile(QUrl &url);
+    QIODevice * handleManifestRequestResponse(QNetworkReply * reply);
+    gpt4all::manifest::ManifestFile *manifest;
+    QNetworkAccessManager m_networkManager;
+    QMap<QNetworkReply*, QFile*> download_tracking;
+    HashFile *saver;
+    QDateTime m_startTime;
+    QString platform_ext;
+    QFile *downloadPath;
+    friend class Downloader;
+};
+
+}
+}

gpt4all-updater/include/Embedded.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+
+#include <QCoreApplication>
+#include <QFile>
+#include <QByteArray>
+
+// Symbols indicating beginning and end of embedded installer
+extern "C" {
+    extern char data_start_gpt4all_installer, data_stop_gpt4all_installer;
+    extern int gpt4all_installer_size;
+}
+
+namespace gpt4all {
+namespace embedded {
+
+class EmbeddedInstaller : public QObject
+{
+public:
+    EmbeddedInstaller();
+    void extractAndDecode();
+    void installInstaller();
+
+private:
+    char * start;
+    char * end;
+    int size;
+    QByteArray data;
+};
+
+
+}
+}
+

gpt4all-updater/include/Manifest.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "Package.h"
+
+#include <QVersionNumber>
+#include <QDate>
+#include <QFile>
+#include <QXmlStreamReader>
+#include <QString>
+#include <QStringList>
+
+namespace gpt4all{
+namespace manifest {
+
+enum releaseType{
+    RELEASE,
+    DEBUG
+};
+
+class ManifestFile {
+public:
+    static ManifestFile * parseManifest(QIODevice *manifestInput);
+    QUrl & getInstallerEndpoint();
+    QString & getExpectedHash();
+private:
+    ManifestFile() {}
+    QString name;
+    QVersionNumber release_ver;
+    QString notes;
+    QStringList authors;
+    QDate release_date;
+    releaseType config;
+    QVersionNumber last_supported_version;
+    QString entity;
+    QStringList component_list;
+    Package pkg;
+    void parsePkgDescription();
+    void parsePkgManifest();
+    QXmlStreamReader xml;
+};
+
+
+
+
+}
+}

gpt4all-updater/include/Modify.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "Command.h"
+
+
+namespace gpt4all {
+namespace modify {
+
+class Modify : public gpt4all::command::Command
+{
+public:
+    Modify(QFile &installer);
+};
+}
+}

gpt4all-updater/include/Package.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include <QCryptographicHash>
+#include <QXmlStreamReader>
+#include <QString>
+#include <QStringList>
+#include <QUrl>
+
+namespace gpt4all {
+namespace manifest {
+
+class Package {
+public:
+    Package() {}
+    static Package parsePackage(QXmlStreamReader &xml);
+
+    QString checksum_sha256;
+    bool is_signed;
+    QUrl installer_endpoint;
+    QUrl sbom_manifest;
+    QStringList installer_args;
+};
+
+}
+
+}