Fix 404 during project reset test

backend/LexBoxApi/Jobs/UpdateProjectMetadataJob.cs

@@ -21,6 +21,6 @@ await QueueJob(schedulerFactory,
     protected override async Task ExecuteJob(IJobExecutionContext context)
     {
         ArgumentException.ThrowIfNullOrEmpty(ProjectCode);
-        await projectService.UpdateProjectMetadata(ProjectCode);
+        await projectService.UpdateProjectMetadataForCode(ProjectCode);
     }
 }

backend/LexBoxApi/Services/HgService.cs

@@ -21,6 +21,7 @@ namespace LexBoxApi.Services;
 public partial class HgService : IHgService
 {
     private const string DELETED_REPO_FOLDER = "_____deleted_____";
+    private const string TEMP_REPO_FOLDER = "_____temp_____";
 
     private readonly IOptions<HgConfig> _options;
     private readonly Lazy<HttpClient> _hgClient;
@@ -38,6 +39,7 @@ public HgService(IOptions<HgConfig> options, IHttpClientFactory clientFactory, I
 
     public static string PrefixRepoRequestPath(string code) => $"{code[0]}/{code}";
     private string PrefixRepoFilePath(string code) => Path.Combine(_options.Value.RepoPath, code[0].ToString(), code);
+    private string GetTempRepoPath(string code, string reason) => Path.Combine(_options.Value.RepoPath, TEMP_REPO_FOLDER, $"{code}__{reason}__{FileUtils.ToTimestamp(DateTimeOffset.UtcNow)}");
 
     private async Task<HttpResponseMessage> GetResponseMessage(string code, string requestPath)
     {
@@ -61,12 +63,14 @@ public async Task InitRepo(string code)
         AssertIsSafeRepoName(code);
         if (Directory.Exists(PrefixRepoFilePath(code)))
             throw new AlreadyExistsException($"Repo already exists: {code}.");
-        await Task.Run(() => InitRepoAt(code));
+        await Task.Run(() =>
+        {
+            InitRepoAt(new DirectoryInfo(PrefixRepoFilePath(code)));
+        });
     }
 
-    private void InitRepoAt(string code)
+    private void InitRepoAt(DirectoryInfo repoDirectory)
     {
-        var repoDirectory = new DirectoryInfo(PrefixRepoFilePath(code));
         repoDirectory.Create();
         FileUtils.CopyFilesRecursively(
             new DirectoryInfo("Services/HgEmptyRepo"),
@@ -95,45 +99,51 @@ public async Task DeleteRepo(string code)
 
     public async Task ResetRepo(string code)
     {
-        string timestamp = FileUtils.ToTimestamp(DateTimeOffset.UtcNow);
-        await SoftDeleteRepo(code, $"{timestamp}__reset");
+        var tmpRepo = new DirectoryInfo(GetTempRepoPath(code, "reset"));
+        InitRepoAt(tmpRepo);
+        await SoftDeleteRepo(code, $"{FileUtils.ToTimestamp(DateTimeOffset.UtcNow)}__reset");
         //we must init the repo as uploading a zip is optional
-        await InitRepo(code);
+        tmpRepo.MoveTo(PrefixRepoFilePath(code));
     }
 
     public async Task FinishReset(string code, Stream zipFile)
     {
-        using var archive = new ZipArchive(zipFile, ZipArchiveMode.Read);
-        await DeleteRepo(code);
-        var repoPath = PrefixRepoFilePath(code);
-        var dir = Directory.CreateDirectory(repoPath);
-        archive.ExtractToDirectory(repoPath);
+        var tempRepoPath = GetTempRepoPath(code, "upload");
+        var tempRepo = Directory.CreateDirectory(tempRepoPath);
+        // TODO: Is Task.Run superfluous here? Or a good idea? Don't know the ins and outs of what happens before the first await in an async method in ASP.NET Core...
+        await Task.Run(() =>
+        {
+            using var archive = new ZipArchive(zipFile, ZipArchiveMode.Read);
+            archive.ExtractToDirectory(tempRepoPath);
+        });
 
-        var hgPath = Path.Join(repoPath, ".hg");
+        var hgPath = Path.Join(tempRepoPath, ".hg");
         if (!Directory.Exists(hgPath))
         {
-            var hgFolder = Directory.EnumerateDirectories(repoPath, ".hg", SearchOption.AllDirectories)
+            var hgFolder = Directory.EnumerateDirectories(tempRepoPath, ".hg", SearchOption.AllDirectories)
                 .FirstOrDefault();
             if (hgFolder is null)
             {
-                await DeleteRepo(code);
-                await InitRepo(code); // we don't want 404s
+                // Don't want to leave invalid .zip contents lying around as they may have been quite large
+                Directory.Delete(tempRepoPath, true);
                 //not sure if this is the best way to handle this, might need to catch it further up to expose the error properly to tus
                 throw ProjectResetException.ZipMissingHgFolder();
             }
             //found the .hg folder, move it to the correct location and continue
             Directory.Move(hgFolder, hgPath);
         }
-        await CleanupRepoFolder(repoPath);
-        SetPermissionsRecursively(dir);
+        await CleanupRepoFolder(tempRepo);
+        SetPermissionsRecursively(tempRepo);
+        // Now we're ready to move the new repo into place, replacing the old one
+        await DeleteRepo(code);
+        tempRepo.MoveTo(PrefixRepoFilePath(code));
     }
 
     /// <summary>
     /// deletes all files and folders in the repo folder except for .hg
     /// </summary>
-    private async Task CleanupRepoFolder(string path)
+    private async Task CleanupRepoFolder(DirectoryInfo repoDir)
     {
-        var repoDir = new DirectoryInfo(path);
         await Task.Run(() =>
         {
             foreach (var info in repoDir.EnumerateFileSystemInfos())
@@ -268,7 +278,7 @@ private async Task<HttpContent> ExecuteHgCommandServerCommand(string code, strin
         return response.Content;
     }
 
-    private static readonly string[] InvalidRepoNames = { DELETED_REPO_FOLDER, "api" };
+    private static readonly string[] InvalidRepoNames = { DELETED_REPO_FOLDER, TEMP_REPO_FOLDER, "api" };
 
     private void AssertIsSafeRepoName(string name)
     {

backend/LexBoxApi/Services/ProjectService.cs

@@ -86,35 +86,44 @@ public async ValueTask<Guid> LookupProjectId(string projectCode)
     public async Task ResetProject(ResetProjectByAdminInput input)
     {
         var rowsAffected = await dbContext.Projects.Where(p => p.Code == input.Code && p.ResetStatus == ResetStatus.None)
-            .ExecuteUpdateAsync(u => u.SetProperty(p => p.ResetStatus, ResetStatus.InProgress));
+            .ExecuteUpdateAsync(u => u
+                .SetProperty(p => p.ResetStatus, ResetStatus.InProgress)
+                .SetProperty(p => p.LastCommit, null as DateTimeOffset?));
         if (rowsAffected == 0) throw new NotFoundException($"project {input.Code} not ready for reset, either already reset or not found");
+        await ResetLexEntryCount(input.Code);
         await hgService.ResetRepo(input.Code);
     }
 
     public async Task FinishReset(string code, Stream? zipFile = null)
     {
-        var project = await dbContext.Projects.Where(p => p.Code == code).SingleOrDefaultAsync();
+        var project = await dbContext.Projects.Include(p => p.FlexProjectMetadata).Where(p => p.Code == code).SingleOrDefaultAsync();
         if (project is null) throw new NotFoundException($"project {code} not found");
         if (project.ResetStatus != ResetStatus.InProgress) throw ProjectResetException.ResetNotStarted(code);
         if (zipFile is not null)
         {
             await hgService.FinishReset(code, zipFile);
-            project.LastCommit = await hgService.GetLastCommitTimeFromHg(project.Code);
+            await UpdateProjectMetadata(project);
         }
         project.ResetStatus = ResetStatus.None;
         project.UpdateUpdatedDate();
         await dbContext.SaveChangesAsync();
     }
 
-    public async Task UpdateProjectMetadata(string projectCode)
+    public async Task UpdateProjectMetadataForCode(string projectCode)
     {
         var project = await dbContext.Projects
             .Include(p => p.FlexProjectMetadata)
             .FirstOrDefaultAsync(p => p.Code == projectCode);
         if (project is null) return;
+        await UpdateProjectMetadata(project);
+        await dbContext.SaveChangesAsync();
+    }
+
+    public async Task UpdateProjectMetadata(Project project)
+    {
         if (hgConfig.Value.AutoUpdateLexEntryCountOnSendReceive && project is { Type: ProjectType.FLEx })
         {
-            var count = await hgService.GetLexEntryCount(projectCode);
+            var count = await hgService.GetLexEntryCount(project.Code);
             if (project.FlexProjectMetadata is null)
             {
                 project.FlexProjectMetadata = new FlexProjectMetadata { LexEntryCount = count };
@@ -125,8 +134,21 @@ public async Task UpdateProjectMetadata(string projectCode)
             }
         }
 
-        project.LastCommit = await hgService.GetLastCommitTimeFromHg(projectCode);
-        await dbContext.SaveChangesAsync();
+        project.LastCommit = await hgService.GetLastCommitTimeFromHg(project.Code);
+        // Caller is responsible for caling dbContext.SaveChangesAsync()
+    }
+
+    public async Task ResetLexEntryCount(string projectCode)
+    {
+        var project = await dbContext.Projects
+            .Include(p => p.FlexProjectMetadata)
+            .FirstOrDefaultAsync(p => p.Code == projectCode);
+        if (project is null) return;
+        if (project.FlexProjectMetadata is not null)
+        {
+            project.FlexProjectMetadata.LexEntryCount = null;
+            await dbContext.SaveChangesAsync();
+        }
     }
 
     public async Task<DateTimeOffset?> UpdateLastCommit(string projectCode)

backend/Testing/ApiTests/ApiTestBase.cs

@@ -41,4 +41,23 @@ public async Task<JsonObject> ExecuteGql([StringSyntax("graphql")] string gql, b
         response.IsSuccessStatusCode.ShouldBeTrue($"code was {(int)response.StatusCode} ({response.ReasonPhrase})");
         return jsonResponse;
     }
+
+    public async Task<string?> GetProjectLastCommit(string projectCode)
+    {
+        var jsonResult = await ExecuteGql($$"""
+query projectLastCommit {
+    projectByCode(code: "{{projectCode}}") {
+        lastCommit
+    }
+}
+""");
+        var project = jsonResult?["data"]?["projectByCode"].ShouldBeOfType<JsonObject>();
+        return project?["lastCommit"]?.ToString();
+    }
+
+    public async Task StartLexboxProjectReset(string projectCode)
+    {
+        var response = await HttpClient.PostAsync($"{BaseUrl}/api/project/resetProject/{projectCode}", null);
+        response.EnsureSuccessStatusCode();
+    }
 }

backend/Testing/ApiTests/ResetProjectRaceConditions.cs

@@ -0,0 +1,76 @@
+using Shouldly;
+using Testing.Fixtures;
+using static Testing.Services.Utils;
+
+namespace Testing.ApiTests;
+
+// Issue: https://github.com/sillsdev/languageforge-lexbox/issues/728
+// Sadly, this does not reproduce the 404, but I still think it's a decent test
+[Trait("Category", "Integration")]
+public class ResetPojectRaceCondition : IClassFixture<IntegrationFixture>
+{
+
+    private readonly IntegrationFixture _fixture;
+    private readonly ApiTestBase _adminApiTester;
+
+    public ResetPojectRaceCondition(IntegrationFixture fixture)
+    {
+        _fixture = fixture;
+        _adminApiTester = _fixture.AdminApiTester;
+    }
+
+    [Fact]
+    public async Task SimultaneousResetsDontResultIn404s()
+    {
+        // Create projects on server
+        var config1 = GetNewProjectConfig();
+        var config2 = GetNewProjectConfig();
+        var config3 = GetNewProjectConfig();
+
+        await using var project1 = await RegisterProjectInLexBox(config1, _adminApiTester);
+        await using var project2 = await RegisterProjectInLexBox(config2, _adminApiTester);
+        await using var project3 = await RegisterProjectInLexBox(config3, _adminApiTester);
+
+        var lastCommitBefore1 = await _adminApiTester.GetProjectLastCommit(config1.Code);
+        var lastCommitBefore2 = await _adminApiTester.GetProjectLastCommit(config2.Code);
+        var lastCommitBefore3 = await _adminApiTester.GetProjectLastCommit(config3.Code);
+
+        lastCommitBefore1.ShouldBeNullOrWhiteSpace();
+        lastCommitBefore2.ShouldBeNullOrWhiteSpace();
+        lastCommitBefore3.ShouldBeNullOrWhiteSpace();
+
+        // Reset and fill projects on server
+        var newLastCommits = await Task.WhenAll(
+            DoFullProjectResetAndVerifyLastCommit(config1.Code),
+            DoFullProjectResetAndVerifyLastCommit(config2.Code),
+            DoFullProjectResetAndVerifyLastCommit(config3.Code)
+        );
+
+        newLastCommits[0].ShouldNotBeNullOrWhiteSpace();
+        newLastCommits[0].ShouldBe(newLastCommits[1]);
+        newLastCommits[0].ShouldBe(newLastCommits[2]);
+
+        // we need a short delay between resets or we'll get naming collisions on the backups of the reset projects
+        await Task.Delay(1000);
+
+        // Reset and fill projects on server
+        var templateRepoLastCommit = newLastCommits[0];
+        newLastCommits = await Task.WhenAll(
+            DoFullProjectResetAndVerifyLastCommit(config1.Code, templateRepoLastCommit),
+            DoFullProjectResetAndVerifyLastCommit(config2.Code, templateRepoLastCommit),
+            DoFullProjectResetAndVerifyLastCommit(config3.Code, templateRepoLastCommit)
+        );
+    }
+
+    private async Task<string?> DoFullProjectResetAndVerifyLastCommit(string projectCode, string? expectedLastCommit = null)
+    {
+        await _adminApiTester.StartLexboxProjectReset(projectCode);
+        var lastCommitBefore = await _adminApiTester.GetProjectLastCommit(projectCode);
+        lastCommitBefore.ShouldBeNullOrWhiteSpace();
+        await _fixture.FinishLexboxProjectResetWithTemplateRepo(projectCode);
+        var lastCommit = await _adminApiTester.GetProjectLastCommit(projectCode);
+        if (expectedLastCommit is not null) lastCommit.ShouldBe(expectedLastCommit);
+        else lastCommit.ShouldNotBeNullOrWhiteSpace();
+        return lastCommit;
+    }
+}

backend/Testing/Fixtures/SendReceiveFixture.cs → backend/Testing/Fixtures/IntegrationFixture.cs

@@ -1,25 +1,31 @@
 using System.IO.Compression;
+using System.Net;
 using System.Runtime.CompilerServices;
 using Chorus.VcsDrivers.Mercurial;
 using LexCore.Utils;
 using Shouldly;
+using SIL.Linq;
 using SIL.Progress;
 using Testing.ApiTests;
 using Testing.Services;
+using TusDotNetClient;
 using static Testing.Services.Constants;
 
 namespace Testing.Fixtures;
 
-public class SendReceiveFixture : IAsyncLifetime
+public class IntegrationFixture : IAsyncLifetime
 {
+    private readonly FileInfo _templateRepoZip = new(Path.Join(BasePath, "_template-repo_.zip"));
     private readonly DirectoryInfo _templateRepo = new(Path.Join(BasePath, "_template-repo_"));
     public ApiTestBase AdminApiTester { get; } = new();
+    private string AdminJwt = string.Empty;
 
     public async Task InitializeAsync()
     {
         DeletePreviousTestFiles();
+        Directory.CreateDirectory(BasePath);
         await DownloadTemplateRepo();
-        await AdminApiTester.LoginAs(AdminAuth.Username, AdminAuth.Password);
+        AdminJwt = await AdminApiTester.LoginAs(AdminAuth.Username, AdminAuth.Password);
     }
 
     public Task DisposeAsync()
@@ -37,6 +43,8 @@ private async Task DownloadTemplateRepo()
         await using var stream = await AdminApiTester.HttpClient.GetStreamAsync("https://drive.google.com/uc?export=download&id=1w357T1Ti7bDwEof4HPBUZ5gB7WSKA5O2");
         using var zip = new ZipArchive(stream);
         zip.ExtractToDirectory(_templateRepo.FullName);
+        InitRepoInDirectory(_templateRepo.FullName);
+        ZipFile.CreateFromDirectory(_templateRepo.FullName, _templateRepoZip.FullName);
     }
 
     public ProjectConfig InitLocalFlexProjectWithRepo(HgProtocol? protocol = null, [CallerMemberName] string projectName = "")
@@ -52,12 +60,29 @@ public void InitLocalFlexProjectWithRepo(ProjectPath projectPath)
         FileUtils.CopyFilesRecursively(_templateRepo, projectDir);
         File.Move(Path.Join(projectPath.Dir, "kevin-test-01.fwdata"), projectPath.FwDataFile);
         Directory.EnumerateFiles(projectPath.Dir).ShouldContain(projectPath.FwDataFile);
+    }
+
+    public async Task FinishLexboxProjectResetWithTemplateRepo(string projectCode)
+    {
+        await FinishLexboxProjectResetWithRepo(projectCode, _templateRepoZip);
+    }
 
-        // hack around the fact that our send and receive won't create a repo from scratch.
+    public async Task FinishLexboxProjectResetWithRepo(string projectCode, FileInfo repo)
+    {
+        var client = new TusClient();
+        client.AdditionalHeaders.Add("Cookie", $".LexBoxAuth={AdminJwt}");
+        var fileUrl = await client.CreateAsync($"{AdminApiTester.BaseUrl}/api/project/upload-zip/{projectCode}", repo.Length, [("filetype", "application/zip")]);
+        var responses = await client.UploadAsync(fileUrl, repo, chunkSize: 20);
+        responses.Select(r => r.StatusCode.ToString())
+            .ForEach((codeName) => codeName.ShouldBe(nameof(HttpStatusCode.NoContent)));
+    }
+
+    private static void InitRepoInDirectory(string projectDir)
+    {
         var progress = new NullProgress();
-        HgRunner.Run("hg init", projectPath.Dir, 5, progress);
-        HgRunner.Run("hg branch 7500002.7000072", projectPath.Dir, 5, progress);
-        HgRunner.Run($"hg add Lexicon.fwstub", projectPath.Dir, 5, progress);
-        HgRunner.Run("""hg commit -m "first commit" """, projectPath.Dir, 5, progress);
+        HgRunner.Run("hg init", projectDir, 5, progress);
+        HgRunner.Run("hg branch 7500002.7000072", projectDir, 5, progress);
+        HgRunner.Run($"hg add Lexicon.fwstub", projectDir, 5, progress);
+        HgRunner.Run("""hg commit -m "first commit" """, projectDir, 5, progress);
     }
 }

backend/Testing/Services/Utils.cs

@@ -22,11 +22,10 @@ public static SendReceiveParams GetParams(HgProtocol protocol,
 
     public static ProjectConfig GetNewProjectConfig(HgProtocol? protocol = null, [CallerMemberName] string projectName = "")
     {
-        var id = Guid.NewGuid();
         if (protocol.HasValue) projectName += $" ({protocol.Value.ToString()[..5]})";
-        var projectCode = ToProjectCodeFriendlyString(projectName);
+        var id = Guid.NewGuid();
         var shortId = id.ToString().Split("-")[0];
-        projectCode = $"{projectCode}-{shortId}-dev-flex";
+        var projectCode = $"{ToProjectCodeFriendlyString(projectName)}-{shortId}-dev-flex";
         var dir = GetNewProjectDir(projectCode, projectName);
         return new ProjectConfig(id, projectName, projectCode, dir);
     }
@@ -72,7 +71,7 @@ public static void ValidateSendReceiveOutput(string srOutput)
 
     public static string ToProjectCodeFriendlyString(string name)
     {
-        var dashesBeforeCapitals = Regex.Replace(name, "(?<!^)([A-Z][a-z]|(?<=[a-z])[A-Z])", "-$1")
+        var dashesBeforeCapitals = Regex.Replace(name, "(?<!^)([A-Z][a-z]|(?<=[a-z])[A-Z0-9]|(?<=[0-9])[A-Z])", "-$1")
             .Trim().ToLower();
         var onlyLettersNumbersAndDashes = Regex.Replace(dashesBeforeCapitals, @"[^a-zA-Z0-9]+", "-");
         return onlyLettersNumbersAndDashes.Trim('-');