#33: Unexpected size for index file [always upload all Packages and Release files]

Aldaviva · Aldaviva · commit 7c79ddea01a2 · 2024-12-07T10:28:55.000-08:00
diff --git a/RaspberryPiDotnetRepository/Data/PackageIndexFile.cs b/RaspberryPiDotnetRepository/Data/PackageIndexFile.cs
@@ -3,7 +3,7 @@ namespace RaspberryPiDotnetRepository.Data;
 /// <summary>
 /// This represents <c>Packages.gz</c> or <c>Packages</c> files in a Debian package repository
 /// </summary>
-public record PackageIndexFile(DebianRelease debianVersion, CpuArchitecture architecture, bool isCompressed, bool isUpToDateInBlobStorage = false) {
+public record PackageIndexFile(DebianRelease debianVersion, CpuArchitecture architecture, bool isCompressed) {
 
     public string filePathRelativeToSuite => Path.Combine("main", $"binary-{architecture.toDebian()}", $"Packages{(isCompressed ? ".gz" : "")}");
     public string filePathRelativeToRepo => Path.Combine("dists", debianVersion.getCodename(), filePathRelativeToSuite);
diff --git a/RaspberryPiDotnetRepository/Data/ReleaseIndexFile.cs b/RaspberryPiDotnetRepository/Data/ReleaseIndexFile.cs
@@ -3,7 +3,7 @@ namespace RaspberryPiDotnetRepository.Data;
 /// <summary>
 /// This represents <c>InRelease</c>, <c>Release</c>, or <c>Release.gpg</c> files in a Debian package repository
 /// </summary>
-public record ReleaseIndexFile(DebianRelease debianVersion, bool isUpToDateInBlobStorage = false) {
+public class ReleaseIndexFile(DebianRelease debianVersion) {
 
     private readonly string parentDirectory = Path.Combine("dists", debianVersion.getCodename());
 
diff --git a/RaspberryPiDotnetRepository/Data/UploadableFile.cs b/RaspberryPiDotnetRepository/Data/UploadableFile.cs
@@ -1,3 +1,3 @@
 namespace RaspberryPiDotnetRepository.Data;
 
-public record UploadableFile(string filePathRelativeToRepo, bool isUpToDateInBlobStorage = false);
+public record UploadableFile(string filePathRelativeToRepo);
diff --git a/RaspberryPiDotnetRepository/Debian/Repository/ExtraFileGenerator.cs b/RaspberryPiDotnetRepository/Debian/Repository/ExtraFileGenerator.cs
@@ -14,6 +14,8 @@ public interface ExtraFileGenerator {
 
     string copyGpgPublicKey();
 
+    Task<string> generateAddRepoScript();
+
 }
 
 public class ExtraFileGeneratorImpl(IOptions<Options> options, StatisticsService statistics, ILogger<ExtraFileGeneratorImpl> logger): ExtraFileGenerator {
@@ -59,8 +61,37 @@ public string copyGpgPublicKey() {
         const string FILENAME                = "aldaviva.gpg.key";
         string       gpgPublicKeyDestination = Path.Combine(options.Value.repositoryBaseDir, FILENAME);
         File.Copy(options.Value.gpgPublicKeyPath, gpgPublicKeyDestination, true);
+        statistics.onFileWritten(gpgPublicKeyDestination);
         logger.LogDebug("Wrote GPG public key file Debian repository");
         return FILENAME;
     }
 
+    public async Task<string> generateAddRepoScript() {
+        const string FILENAME = "addrepo.sh";
+        string       filePath = Path.Combine(options.Value.repositoryBaseDir, FILENAME);
+        const string CONTENTS = """
+                                #!/bin/sh
+
+                                echo Adding repository PGP key
+                                sudo wget -q https://raspbian.aldaviva.com/aldaviva.gpg.key -O /etc/apt/trusted.gpg.d/aldaviva.gpg
+
+                                echo Adding repository
+                                echo "deb https://raspbian.aldaviva.com/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/aldaviva.list > /dev/null
+
+                                echo Finding available packages
+                                sudo apt update
+
+                                echo Ready to install .NET packages, for example:
+                                echo "  sudo apt install dotnet-runtime-latest"
+                                echo "  sudo apt install aspnetcore-runtime-latest-lts"
+                                echo "  sudo apt install dotnet-sdk-8.0"
+                                echo For more information, see https://github.com/Aldaviva/RaspberryPiDotnetRepository
+                                """;
+
+        await File.WriteAllTextAsync(filePath, CONTENTS, Encoding.UTF8);
+        statistics.onFileWritten(filePath);
+        logger.LogDebug("Wrote client-side repository installation script");
+        return FILENAME;
+    }
+
 }
diff --git a/RaspberryPiDotnetRepository/Debian/Repository/Indexer.cs b/RaspberryPiDotnetRepository/Debian/Repository/Indexer.cs
@@ -50,16 +50,13 @@ public async Task<IEnumerable<IGrouping<DebianRelease, PackageIndexFile>>> gener
 
     private async Task<IEnumerable<PackageIndexFile>> generateIndexOfPackagesInDebianReleaseAndArchitecture(DebianRelease debianRelease, CpuArchitecture cpuArchitecture,
                                                                                                             IEnumerable<DebianPackage> debPackages, UpstreamReleasesSecondaryInfo upstreamInfo) {
-        bool areAllPackagesInIndexUpToDateInBlobStorage = true;
-        string packageFileContents = string.Join("\n\n", debPackages.Select(package => {
-            areAllPackagesInIndexUpToDateInBlobStorage &= package.isUpToDateInBlobStorage;
-            return $"""
-                    {package.getControl(upstreamInfo).serialize().Trim()}
-                    Filename: {package.filePathRelativeToRepo}
-                    Size: {package.downloadSize.ConvertToUnit(Unit.Byte).Quantity:F0}
-                    SHA256: {package.fileHashSha256}
-                    """.ReplaceLineEndings("\n");
-        }));
+        string packageFileContents = string.Join("\n\n", debPackages.Select(package =>
+            $"""
+                 {package.getControl(upstreamInfo).serialize().Trim()}
+                 Filename: {package.filePathRelativeToRepo}
+                 Size: {package.downloadSize.ConvertToUnit(Unit.Byte).Quantity:F0}
+                 SHA256: {package.fileHashSha256}
+                 """.ReplaceLineEndings("\n")));
 
         string packageFileRelativeToSuite = Path.Combine("main", $"binary-{cpuArchitecture.toDebian()}", "Packages");
         string packageFileAbsolutePath    = Path.Combine(options.Value.repositoryBaseDir, "dists", debianRelease.getCodename(), packageFileRelativeToSuite);
@@ -79,22 +76,19 @@ private async Task<IEnumerable<PackageIndexFile>> generateIndexOfPackagesInDebia
         logger.LogInformation("Generated Packages index files for Debian {debian} {arch}", debianRelease.getCodename(), cpuArchitecture.toDebian());
 
         return [
-            new PackageIndexFile(debianRelease, cpuArchitecture, true, areAllPackagesInIndexUpToDateInBlobStorage),
-            new PackageIndexFile(debianRelease, cpuArchitecture, false, areAllPackagesInIndexUpToDateInBlobStorage)
+            new PackageIndexFile(debianRelease, cpuArchitecture, true),
+            new PackageIndexFile(debianRelease, cpuArchitecture, false)
         ];
     }
 
     public async Task<ReleaseIndexFile> generateReleaseIndex(DebianRelease debianRelease, IEnumerable<PackageIndexFile> packageIndexFiles) {
-        string repositoryBaseDir                            = options.Value.repositoryBaseDir;
-        bool   areAllReleaseIndexFilesUpToDateInBlobStorage = true;
+        string repositoryBaseDir = options.Value.repositoryBaseDir;
 
         string indexSha256Hashes = string.Join('\n', await Task.WhenAll(packageIndexFiles.Select(async packageIndexFile => {
             await using FileStream fileStream = File.OpenRead(Path.Combine(repositoryBaseDir, packageIndexFile.filePathRelativeToRepo));
 
             string sha256Hash = Convert.ToHexString(await SHA256.HashDataAsync(fileStream)).ToLowerInvariant();
-            long   fileSize   = fileStream.Length;
-            areAllReleaseIndexFilesUpToDateInBlobStorage &= packageIndexFile.isUpToDateInBlobStorage;
-            return $" {sha256Hash} {fileSize:D} {Paths.Dos2UnixSlashes(packageIndexFile.filePathRelativeToSuite)}";
+            return $" {sha256Hash} {fileStream.Length:D} {Paths.Dos2UnixSlashes(packageIndexFile.filePathRelativeToSuite)}";
         })));
 
         string releaseFileCleartext =
@@ -111,7 +105,7 @@ public async Task<ReleaseIndexFile> generateReleaseIndex(DebianRelease debianRel
                  {indexSha256Hashes}
                  """.ReplaceLineEndings("\n");
 
-        ReleaseIndexFile releaseIndexFile = new(debianRelease, areAllReleaseIndexFilesUpToDateInBlobStorage);
+        ReleaseIndexFile releaseIndexFile = new(debianRelease);
 
         string filePath = Path.Combine(repositoryBaseDir, releaseIndexFile.releaseFilePathRelativeToRepo);
         await File.WriteAllTextAsync(filePath, releaseFileCleartext, Encoding.UTF8); // Bom.Squad has already defused this UTF-8 BOM, or else apt will get its limbs blown off
diff --git a/RaspberryPiDotnetRepository/Orchestrator.cs b/RaspberryPiDotnetRepository/Orchestrator.cs
@@ -11,18 +11,18 @@
 namespace RaspberryPiDotnetRepository;
 
 public class Orchestrator(
-    SdkDownloader            sdkDownloader,
-    PackageRequester         packageRequester,
-    PackageGenerator         packageGenerator,
-    Indexer                  indexer,
-    CdnClient                cdnClient,
-    ExtraFileGenerator       extraFileGenerator,
-    ManifestManager          manifestManager,
-    BlobStorageClient        blobStorage,
-    StatisticsService        statistics,
+    SdkDownloader sdkDownloader,
+    PackageRequester packageRequester,
+    PackageGenerator packageGenerator,
+    Indexer indexer,
+    CdnClient cdnClient,
+    ExtraFileGenerator extraFileGenerator,
+    ManifestManager manifestManager,
+    BlobStorageClient blobStorage,
+    StatisticsService statistics,
     IHostApplicationLifetime appLifetime,
-    IOptions<Options>        options,
-    ILogger<Orchestrator>    logger
+    IOptions<Options> options,
+    ILogger<Orchestrator> logger
 ): BackgroundService {
 
     private static readonly Version MIN_DOTNET_MINOR_VERSION = new(6, 0);
@@ -55,42 +55,42 @@ protected override async Task ExecuteAsync(CancellationToken ct) {
             ReleaseIndexFile[] releaseIndexFiles = await Task.WhenAll(packageIndicesByDebianRelease.Select(releaseFiles => indexer.generateReleaseIndex(releaseFiles.Key, releaseFiles)));
 
             // Write readme, badges, and GPG public key
-            string                      readmeFilename   = await extraFileGenerator.generateReadme();
-            IEnumerable<UploadableFile> badgeFiles       = await extraFileGenerator.generateReadmeBadges(upstreamReleases.First(release => release.isLatestMinorVersion));
-            string                      gpgPublicKeyFile = extraFileGenerator.copyGpgPublicKey();
+            string                      readmeFilename    = await extraFileGenerator.generateReadme();
+            IEnumerable<UploadableFile> badgeFiles        = await extraFileGenerator.generateReadmeBadges(upstreamReleases.First(release => release.isLatestMinorVersion));
+            string                      gpgPublicKeyFile  = extraFileGenerator.copyGpgPublicKey();
+            string                      addRepoScriptFile = await extraFileGenerator.generateAddRepoScript();
 
             // Upload .deb packages to Azure Blob Storage
             string repoBaseDir = options.Value.repositoryBaseDir;
             await Task.WhenAll(generatedPackages.Where(p => !p.isUpToDateInBlobStorage).Select(p =>
                 blobStorage.uploadFile(Path.Combine(repoBaseDir, p.filePathRelativeToRepo), p.filePathRelativeToRepo, "application/vnd.debian.binary-package", ct)));
 
             // Upload Packages.gz indices to Azure Blob Storage
-            Task<BlobContentInfo?[]> packageIndexUploads = Task.WhenAll(packageIndicesByDebianRelease.SelectMany(debianRelease => debianRelease).Where(file => !file.isUpToDateInBlobStorage).Select(
-                packageIndexFile =>
-                    blobStorage.uploadFile(Path.Combine(repoBaseDir, packageIndexFile.filePathRelativeToRepo), packageIndexFile.filePathRelativeToRepo,
-                        packageIndexFile.isCompressed ? "application/gzip" : "text/plain", ct)));
+            Task<BlobContentInfo?[]> packageIndexUploads = Task.WhenAll(packageIndicesByDebianRelease.SelectMany(debianRelease => debianRelease).Select(packageIndexFile =>
+                blobStorage.uploadFile(Path.Combine(repoBaseDir, packageIndexFile.filePathRelativeToRepo), packageIndexFile.filePathRelativeToRepo,
+                    packageIndexFile.isCompressed ? "application/gzip" : "text/plain", ct)));
 
-            // Upload InRelease index files to Azure Blob Storage
-            Task<BlobContentInfo?[]> releaseIndexUploads = Task.WhenAll(releaseIndexFiles.Where(file => !file.isUpToDateInBlobStorage).SelectMany(file =>
+            // Upload InRelease indices to Azure Blob Storage
+            Task<BlobContentInfo?[]> releaseIndexUploads = Task.WhenAll(releaseIndexFiles.SelectMany(file =>
                 new[] { file.inreleaseFilePathRelativeToRepo, file.releaseFilePathRelativeToRepo, file.releaseGpgFilePathRelativeToRepo }.Select(relativeFilePath =>
-                    blobStorage.uploadFile(Path.Combine(repoBaseDir, relativeFilePath), relativeFilePath, Path.GetExtension(relativeFilePath) == ".gpg" ? "application/pgp-signature" : "text/plain",
-                        ct))));
+                    blobStorage.uploadFile(Path.Combine(repoBaseDir, relativeFilePath), relativeFilePath,
+                        ".gpg".Equals(Path.GetExtension(relativeFilePath), StringComparison.OrdinalIgnoreCase) ? "application/pgp-signature" : "text/plain", ct))));
 
             await packageIndexUploads;
             await releaseIndexUploads;
 
-            // Upload badge JSON files to Azure Blob Storage
-            await Task.WhenAll(badgeFiles.Where(file => !file.isUpToDateInBlobStorage)
-                .Select(file => blobStorage.uploadFile(Path.Combine(repoBaseDir, file.filePathRelativeToRepo), file.filePathRelativeToRepo, "application/json", ct)));
+            // Upload badge JSON and other extra files to Azure Blob Storage
+            await Task.WhenAll(badgeFiles.Select(file => blobStorage.uploadFile(Path.Combine(repoBaseDir, file.filePathRelativeToRepo), file.filePathRelativeToRepo, "application/json", ct)));
             await blobStorage.uploadFile(Path.Combine(repoBaseDir, readmeFilename), readmeFilename, "text/plain", ct);
             await blobStorage.uploadFile(Path.Combine(repoBaseDir, gpgPublicKeyFile), gpgPublicKeyFile, "application/pgp-keys", ct);
-
-            // Clear CDN cache
-            await cdnClient.purge(["/dists/*", "/badges/*", "/manifest.json"]);
+            await blobStorage.uploadFile(Path.Combine(repoBaseDir, addRepoScriptFile), addRepoScriptFile, "application/x-sh", ct);
 
             // Upload manifest.json file to Azure Blob Storage
             await blobStorage.uploadFile(manifestManager.manifestFilePath, manifestManager.manifestFilename, "application/json", ct);
 
+            // Clear CDN cache
+            await cdnClient.purge(["/dists/*", "/badges/*", "/manifest.json"]);
+
             // Delete outdated .deb package files from Azure Blob Storage
             await Task.WhenAll(oldManifest?.packages.Except(newManifest.packages).Select(packageToDelete => blobStorage.deleteFile(packageToDelete.filePathRelativeToRepo, ct)) ?? []);
 
diff --git a/RaspberryPiDotnetRepository/Program.cs b/RaspberryPiDotnetRepository/Program.cs
@@ -17,14 +17,15 @@
 using PGP = Unfucked.PGP;
 
 #pragma warning disable CS8634 // The type cannot be used as type parameter in the generic type or method. Nullability of type argument doesn't match 'class' constraint. - You can safely try to register a singleton with a null value, it just won't register. Just make sure you inject it as nullable.
+#pragma warning disable IDE0001 // same as above
 
 // apt will get its limbs blown off with "Clearsigned file isn't valid, got 'NOSPLIT' (does the network require authentication?)" if InRelease starts with UTF-8 BOM
 BomSquad.DefuseUtf8Bom();
 
 HostApplicationBuilder appConfig = Host.CreateApplicationBuilder(args);
 appConfig.Configuration.AlsoSearchForJsonFilesInExecutableDirectory();
 
-appConfig.Logging.AddUnfuckedConsole(options => options.IncludeNamespaces = false);
+appConfig.Logging.AddUnfuckedConsole();
 
 appConfig.Services
     .Configure<Options>(appConfig.Configuration)
diff --git a/RaspberryPiDotnetRepository/RaspberryPiDotnetRepository.csproj b/RaspberryPiDotnetRepository/RaspberryPiDotnetRepository.csproj
@@ -5,7 +5,7 @@
         <RuntimeIdentifier>win-x64</RuntimeIdentifier>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <Version>1.1.0</Version>
+        <Version>1.1.1</Version>
         <Authors>Ben Hutchison</Authors>
         <Copyright>© 2024 $(Authors)</Copyright>
         <Company>$(Authors)</Company>
@@ -14,6 +14,8 @@
         <RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
         <ApplicationIcon>rpi-dotnet.ico</ApplicationIcon>
         <NoWarn>$(NoWarn);IDE0079;8524</NoWarn>
+        <PublishSingleFile>true</PublishSingleFile>
+        <SelfContained>false</SelfContained>
     </PropertyGroup>
 
     <ItemGroup>
diff --git a/RaspberryPiDotnetRepository/packages.lock.json b/RaspberryPiDotnetRepository/packages.lock.json

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`namespace RaspberryPiDotnetRepository.Data;`
`2`	`2`
`3`		`-public record UploadableFile(string filePathRelativeToRepo, bool isUpToDateInBlobStorage = false);`
	`3`	`+public record UploadableFile(string filePathRelativeToRepo);`