diff --git a/implement/elm-fullstack/Pine/CacheByFileName.cs b/implement/elm-fullstack/Pine/CacheByFileName.cs index fc0c7deb..3e832b88 100644 --- a/implement/elm-fullstack/Pine/CacheByFileName.cs +++ b/implement/elm-fullstack/Pine/CacheByFileName.cs @@ -4,7 +4,10 @@ namespace Pine; public record CacheByFileName(string CacheDirectory) { - public byte[] GetOrUpdate(string fileName, System.Func getNew) + public byte[] GetOrUpdate(string fileName, System.Func getNew) => + GetOrTryAdd(fileName, getNew)!; + + public byte[]? GetOrTryAdd(string fileName, System.Func tryBuild) { var cacheFilePath = Path.Combine(CacheDirectory, fileName); @@ -17,20 +20,23 @@ public byte[] GetOrUpdate(string fileName, System.Func getNew) catch { } } - var file = getNew(); + var file = tryBuild(); - try + if (file is not null) { - var directory = Path.GetDirectoryName(cacheFilePath); + try + { + var directory = Path.GetDirectoryName(cacheFilePath); - if (directory != null) - Directory.CreateDirectory(directory); + if (directory != null) + Directory.CreateDirectory(directory); - File.WriteAllBytes(cacheFilePath, file); - } - catch (System.Exception e) - { - System.Console.WriteLine("Failed to write cache entry: " + e.ToString()); + File.WriteAllBytes(cacheFilePath, file); + } + catch (System.Exception e) + { + System.Console.WriteLine("Failed to write cache entry: " + e.ToString()); + } } return file; diff --git a/implement/elm-fullstack/Pine/LoadFromGitHubOrGitLab.cs b/implement/elm-fullstack/Pine/LoadFromGitHubOrGitLab.cs index 87f0630b..ed902291 100644 --- a/implement/elm-fullstack/Pine/LoadFromGitHubOrGitLab.cs +++ b/implement/elm-fullstack/Pine/LoadFromGitHubOrGitLab.cs @@ -109,7 +109,7 @@ static public Result LoadFromUrl(string sourceUrl) = static public Result LoadFromUrl( string sourceUrl, - Func, ReadOnlyMemory>> getRepositoryFilesPartialForCommit) + Func, ReadOnlyMemory>>> getRepositoryFilesPartialForCommit) { var parsedUrl = ParseUrl(sourceUrl); @@ -129,13 +129,12 @@ static public Result LoadFromUrl( var cloneUrl = parsedUrl.repository.TrimEnd('/') + ".git"; - if (getRepositoryFilesPartialForCommit == null) - getRepositoryFilesPartialForCommit = - req => GetRepositoryFilesPartialForCommitViaLibGitSharpCheckout( - cloneUrl: req.cloneUrlCandidates[0], - commit: req.commit); + getRepositoryFilesPartialForCommit ??= + req => GetRepositoryFilesPartialForCommitViaLibGitSharpCheckout( + cloneUrl: req.cloneUrlCandidates[0], + commit: req.commit); - var repositoryFilesPartial = + var repositoryFilesResult = refLooksLikeCommit ? getRepositoryFilesPartialForCommit( new GetRepositoryFilesPartialForCommitRequest( @@ -147,207 +146,231 @@ static public Result LoadFromUrl( var tempWorkingDirectory = Filesystem.CreateRandomDirectoryInTempDirectory(); var gitRepositoryLocalDirectory = Path.Combine(tempWorkingDirectory, "git-repository"); - try - { - foreach (var fileWithPath in repositoryFilesPartial) + return + repositoryFilesResult + .AndThen(repositoryFilesPartial => { - var absoluteFilePath = Path.Combine(new[] { gitRepositoryLocalDirectory }.Concat(fileWithPath.Key).ToArray()); - var absoluteDirectoryPath = Path.GetDirectoryName(absoluteFilePath)!; - - Directory.CreateDirectory(absoluteDirectoryPath); - File.WriteAllBytes(absoluteFilePath, fileWithPath.Value.ToArray()); - } - - (string hash, CommitContent content)? rootCommit = null; - - using var gitRepository = new Repository(gitRepositoryLocalDirectory); - - Commit? startCommit = null; - - if (parsedUrl.inRepository == null) - { - startCommit = gitRepository.Head.Commits.FirstOrDefault(); - - if (startCommit == null) - return Result.err( - "Failed to get the first commit from HEAD"); - } - else - { - startCommit = gitRepository.Lookup(parsedUrl.inRepository.@ref) as Commit; - - if (startCommit == null) - return Result.err( - "I did not find the commit for ref '" + parsedUrl.inRepository.@ref + "'."); - } - - ParsedUrlInRepository partInRepositoryWithCommit(Commit replacementCommit) => - parsedUrl.inRepository == null ? - new ParsedUrlInRepository(GitObjectType.tree, @ref: replacementCommit.Sha, path: "") : - parsedUrl.inRepository with { @ref = replacementCommit.Sha }; - - var urlInCommit = BackToUrl(parsedUrl with { inRepository = partInRepositoryWithCommit(startCommit) }); - - rootCommit = GetCommitHashAndContent(startCommit); - - var parsedUrlPath = - parsedUrl.inRepository == null ? "" : parsedUrl.inRepository.path; - - var pathNodesNames = parsedUrlPath.Split('/', StringSplitOptions.RemoveEmptyEntries); - - var findGitObjectResult = - FindGitObjectAtPath(startCommit.Tree, pathNodesNames); - - return - findGitObjectResult - .MapError(_ => "I did not find an object at path '" + parsedUrlPath + "' in " + startCommit.Sha) - .AndThen(linkedObject => + try { - IEnumerable traceBackTreeParents() + foreach (var fileWithPath in repositoryFilesPartial) { - var queue = new Queue(); + var absoluteFilePath = Path.Combine(new[] { gitRepositoryLocalDirectory }.Concat(fileWithPath.Key).ToArray()); + var absoluteDirectoryPath = Path.GetDirectoryName(absoluteFilePath)!; - queue.Enqueue(startCommit); - - while (queue.TryDequeue(out var currentCommit)) - { - yield return currentCommit; - - foreach (var parent in currentCommit.Parents) - { - if (FindGitObjectAtPath(parent.Tree, pathNodesNames).Map(find => (string?)find.Sha).WithDefault(null) != linkedObject?.Sha) - continue; - - queue.Enqueue(parent); - } - } + Directory.CreateDirectory(absoluteDirectoryPath); + File.WriteAllBytes(absoluteFilePath, fileWithPath.Value.ToArray()); } - var firstParentCommitWithSameTreeRef = - traceBackTreeParents().OrderBy(commit => commit.Author.When).First(); - - var firstParentCommitWithSameTree = - GetCommitHashAndContent(firstParentCommitWithSameTreeRef); + (string hash, CommitContent content)? rootCommit = null; - var urlInFirstParentCommitWithSameValueAtThisPath = - BackToUrl(parsedUrl with { inRepository = partInRepositoryWithCommit(firstParentCommitWithSameTreeRef) }); + using var gitRepository = new Repository(gitRepositoryLocalDirectory); - static TreeNodeWithStringPath convertToLiteralNodeObjectRecursive(GitObject gitObject) - { - if (gitObject is Tree gitTree) + var loadStartCommitResult = + GetCommitFromReference( + cloneUrl: cloneUrl, + RefCanonicalNameFromPathComponentInGitHubRepository(parsedUrl.inRepository?.@ref)) + .MapError(err => "I did not find the commit for ref '" + err + "'.") + .AndThen(commitId => { - return TreeNodeWithStringPath.SortedTree( - treeContent: - gitTree.Select(treeEntry => - (treeEntry.Name, - convertToLiteralNodeObjectRecursive(treeEntry.Target))) - .ToImmutableList()); - } - - if (gitObject is Blob gitBlob) - { - var memoryStream = new MemoryStream(); - - var gitBlobContentStream = gitBlob.GetContentStream(); - - if (gitBlobContentStream == null) - throw new Exception("Failed to get content of git blob"); - - gitBlobContentStream.CopyTo(memoryStream); + if (gitRepository.Lookup(commitId) is Commit commit) + return Result.ok(commit); - var blobContent = memoryStream.ToArray(); + return Result.err("Did not find commit " + commitId); + }); - var expectedSHA = gitBlob.Sha.ToLowerInvariant(); + return + loadStartCommitResult + .AndThen(startCommit => + { + ParsedUrlInRepository partInRepositoryWithCommit(Commit replacementCommit) => + parsedUrl.inRepository == null ? + new ParsedUrlInRepository(GitObjectType.tree, @ref: replacementCommit.Sha, path: "") : + parsedUrl.inRepository with { @ref = replacementCommit.Sha }; - // This will change with the introduction of the new hash in git. - // We could branch on the length of 'gitBlob.Sha' to choose between old and new hash. - // (https://github.com/git/git/blob/74583d89127e21255c12dd3c8a3bf60b497d7d03/Documentation/technical/hash-function-transition.txt) - // (https://www.youtube.com/watch?v=qHERDFUSa14) - var loadedBlobSHA1Base16Lower = - BitConverter.ToString(GitBlobSHAFromBlobContent(blobContent)).Replace("-", "") - .ToLowerInvariant(); + var urlInCommit = BackToUrl(parsedUrl with { inRepository = partInRepositoryWithCommit(startCommit) }); - if (loadedBlobSHA1Base16Lower != expectedSHA) - throw new Exception("Unexpected content for git object : SHA is " + loadedBlobSHA1Base16Lower + " instead of " + expectedSHA); + rootCommit = GetCommitHashAndContent(startCommit); - return TreeNodeWithStringPath.Blob(memoryStream.ToArray()); - } + var parsedUrlPath = parsedUrl.inRepository == null ? "" : parsedUrl.inRepository.path; - throw new Exception("Unexpected kind of git object: " + gitObject.GetType() + ", " + gitObject.Id); - } + var pathNodesNames = parsedUrlPath.Split('/', StringSplitOptions.RemoveEmptyEntries); + return + FindGitObjectAtPath(startCommit.Tree, pathNodesNames) + .MapError(_ => "I did not find an object at path '" + parsedUrlPath + "' in " + startCommit.Sha) + .AndThen(linkedObject => + { + IEnumerable traceBackTreeParents() + { + var queue = new Queue(); + + queue.Enqueue(startCommit); + + while (queue.TryDequeue(out var currentCommit)) + { + yield return currentCommit; + + foreach (var parent in currentCommit.Parents) + { + if (FindGitObjectAtPath(parent.Tree, pathNodesNames).Map(find => (string?)find.Sha).WithDefault(null) != linkedObject?.Sha) + continue; + + queue.Enqueue(parent); + } + } + } + + var firstParentCommitWithSameTreeRef = + traceBackTreeParents().OrderBy(commit => commit.Author.When).First(); + + var firstParentCommitWithSameTree = + GetCommitHashAndContent(firstParentCommitWithSameTreeRef); + + var urlInFirstParentCommitWithSameValueAtThisPath = + BackToUrl(parsedUrl with { inRepository = partInRepositoryWithCommit(firstParentCommitWithSameTreeRef) }); + + static TreeNodeWithStringPath convertToLiteralNodeObjectRecursive(GitObject gitObject) + { + if (gitObject is Tree gitTree) + { + return TreeNodeWithStringPath.SortedTree( + treeContent: + gitTree.Select(treeEntry => + (treeEntry.Name, + convertToLiteralNodeObjectRecursive(treeEntry.Target))) + .ToImmutableList()); + } + + if (gitObject is Blob gitBlob) + { + var memoryStream = new MemoryStream(); + + var gitBlobContentStream = gitBlob.GetContentStream(); + + if (gitBlobContentStream == null) + throw new Exception("Failed to get content of git blob"); + + gitBlobContentStream.CopyTo(memoryStream); + + var blobContent = memoryStream.ToArray(); + + var expectedSHA = gitBlob.Sha.ToLowerInvariant(); + + // This will change with the introduction of the new hash in git. + // We could branch on the length of 'gitBlob.Sha' to choose between old and new hash. + // (https://github.com/git/git/blob/74583d89127e21255c12dd3c8a3bf60b497d7d03/Documentation/technical/hash-function-transition.txt) + // (https://www.youtube.com/watch?v=qHERDFUSa14) + var loadedBlobSHA1Base16Lower = + BitConverter.ToString(GitBlobSHAFromBlobContent(blobContent)).Replace("-", "") + .ToLowerInvariant(); + + if (loadedBlobSHA1Base16Lower != expectedSHA) + throw new Exception("Unexpected content for git object : SHA is " + loadedBlobSHA1Base16Lower + " instead of " + expectedSHA); + + return TreeNodeWithStringPath.Blob(memoryStream.ToArray()); + } + + throw new Exception("Unexpected kind of git object: " + gitObject.GetType() + ", " + gitObject.Id); + } + + try + { + var literalNodeObject = convertToLiteralNodeObjectRecursive(linkedObject); + + return Result.ok( + new LoadFromUrlSuccess + ( + tree: literalNodeObject, + urlInCommit: urlInCommit, + urlInFirstParentCommitWithSameValueAtThisPath: urlInFirstParentCommitWithSameValueAtThisPath, + rootCommit: rootCommit.Value, + firstParentCommitWithSameTree: firstParentCommitWithSameTree + ) + ); + } + catch (Exception e) + { + return Result.err("Failed to convert from git object:\n" + e.ToString()); + } + }); + }); + } + finally + { try { - var literalNodeObject = convertToLiteralNodeObjectRecursive(linkedObject); - - return Result.ok( - new LoadFromUrlSuccess - ( - tree: literalNodeObject, - urlInCommit: urlInCommit, - urlInFirstParentCommitWithSameValueAtThisPath: urlInFirstParentCommitWithSameValueAtThisPath, - rootCommit: rootCommit.Value, - firstParentCommitWithSameTree: firstParentCommitWithSameTree - ) - ); + DeleteLocalDirectoryRecursive(tempWorkingDirectory); } - catch (Exception e) + catch { - return Result.err("Failed to convert from git object:\n" + e.ToString()); + /* + Adapt to observations 2020-02-15: + A user got a `System.IO.DirectoryNotFoundException` out of `DeleteLocalDirectoryRecursive`. + Also, it seems common other software interferring with contents of `Path.GetTempPath()` (https://github.com/dotnet/runtime/issues/3778). + */ } - }); - } - finally - { - try - { - DeleteLocalDirectoryRecursive(tempWorkingDirectory); - } - catch - { - /* - Adapt to observations 2020-02-15: - A user got a `System.IO.DirectoryNotFoundException` out of `DeleteLocalDirectoryRecursive`. - Also, it seems common other software interferring with contents of `Path.GetTempPath()` (https://github.com/dotnet/runtime/issues/3778). - */ - } - } + } + }); } - static IImmutableDictionary, ReadOnlyMemory> GetRepositoryFilesPartialForCommitDefault(GetRepositoryFilesPartialForCommitRequest request) + static Result, ReadOnlyMemory>> GetRepositoryFilesPartialForCommitDefault( + GetRepositoryFilesPartialForCommitRequest request) { - var getNew = () => - { - foreach (var cloneUrlCandidate in request.cloneUrlCandidates) + var loadNew = + () => { - try - { - return GetRepositoryFilesPartialForCommitViaLibGitSharpCheckout( + var loadCandidates = + request.cloneUrlCandidates.Select( + cloneUrlCandidate => + new Func, ReadOnlyMemory>>>( + () => GetRepositoryFilesPartialForCommitViaLibGitSharpCheckout( cloneUrl: cloneUrlCandidate, - commit: request.commit); - } - catch { } - } + commit: request.commit))); - return null; - }; + return + loadCandidates.FirstOkOrErrors().MapError( + candidatesErrors => "Failed for " + candidatesErrors.Count + " clone urls:\n" + string.Join("\n", candidatesErrors)); + }; - var cache = RepositoryFilesPartialForCommitCacheDefault; + var localCache = new Lazy, ReadOnlyMemory>>>(loadNew); + + var externalCache = RepositoryFilesPartialForCommitCacheDefault; + + var fromExternalCache = + externalCache?.GetOrTryAdd( + fileName: request.commit, + tryBuild: () => + { + return localCache.Value.Unpack( + fromErr: _ => null, + fromOk: files => (byte[]?)ZipArchive.ZipArchiveFromEntries(files)); + }); - if (cache != null) + return fromExternalCache switch { - return + not null => Result, ReadOnlyMemory>>.ok( Composition.ToFlatDictionaryWithPathComparer( Composition.SortedTreeFromSetOfBlobsWithCommonFilePath( - ZipArchive.EntriesFromZipArchive( - cache.GetOrUpdate(request.commit, () => ZipArchive.ZipArchiveFromEntries(getNew())))) - .EnumerateBlobsTransitive()); - } + ZipArchive.EntriesFromZipArchive(fromExternalCache)) + .EnumerateBlobsTransitive())), - return getNew(); + _ => localCache.Value + }; } - static public IImmutableDictionary, ReadOnlyMemory> GetRepositoryFilesPartialForCommitViaLibGitSharpCheckout( + static public Result, ReadOnlyMemory>> GetRepositoryFilesPartialForBranchViaLibGitSharpCheckout( + string cloneUrl, + string? branchName) + { + var refName = RefCanonicalNameFromPathComponentInGitHubRepository(branchName); + + return + GetCommitFromReference(cloneUrl, refName) + .MapError(err => "Failed to get commit from ref '" + refName + "': " + err) + .AndThen(commitId => GetRepositoryFilesPartialForCommitViaLibGitSharpCheckout(cloneUrl, commitId)); + } + + static public Result, ReadOnlyMemory>> GetRepositoryFilesPartialForCommitViaLibGitSharpCheckout( string cloneUrl, string commit) { @@ -367,9 +390,10 @@ static public IImmutableDictionary, ReadOnlyMemory> gitRepository.CheckoutPaths(commit, ImmutableList.Create(tempWorkingDirectory.TrimEnd('/') + "/")); return - Composition.ToFlatDictionaryWithPathComparer( - Filesystem.GetAllFilesFromDirectory(tempWorkingDirectory) - .Where(predicate: c => c.path?[0] == ".git")); + Result, ReadOnlyMemory>>.ok( + Composition.ToFlatDictionaryWithPathComparer( + Filesystem.GetAllFilesFromDirectory(tempWorkingDirectory) + .Where(predicate: c => c.path?[0] == ".git"))); } catch (Exception e) { @@ -381,6 +405,59 @@ static public IImmutableDictionary, ReadOnlyMemory> } } + static public string RefCanonicalNameFromPathComponentInGitHubRepository(string? refPathComponent) + { + if (refPathComponent is null) + return "HEAD"; + + if (RefLooksLikeCommit(refPathComponent)) + return refPathComponent; + + return "refs/heads/" + refPathComponent; + } + + static public Result GetCommitFromReference( + string cloneUrl, + string referenceCanonicalName) + { + if (RefLooksLikeCommit(referenceCanonicalName)) + return Result.ok(referenceCanonicalName); + + var remoteReferences = Repository.ListRemoteReferences(cloneUrl).ToImmutableList(); + + return GetCommitFromReference( + ImmutableHashSet.Empty, + remoteReferences, + referenceCanonicalName); + } + + static public Result GetCommitFromReference( + IImmutableSet stack, + IReadOnlyList remoteReferences, + string referenceCanonicalName) + { + if (RefLooksLikeCommit(referenceCanonicalName)) + return Result.ok(referenceCanonicalName); + + if (stack.Contains(referenceCanonicalName)) + return Result.err("Cyclic reference: '" + referenceCanonicalName + "'"); + + var matchingReference = + remoteReferences.FirstOrDefault(c => c.CanonicalName == referenceCanonicalName); + + if (matchingReference == null) + { + return Result.err("Found no reference matching '" + referenceCanonicalName + "' (" + remoteReferences.Count + " remote references)"); + } + + return GetCommitFromReference( + stack.Add(referenceCanonicalName), + remoteReferences, + matchingReference.TargetIdentifier); + } + + static public bool RefLooksLikeCommit(string reference) => Regex.IsMatch(reference, "[A-Fa-f0-9]{40}"); + static public IImmutableDictionary, ReadOnlyMemory> GetRepositoryFilesPartialForCommitViaEnvironmentGitCheckout( string cloneUrl, string commit) @@ -448,34 +525,6 @@ static public IImmutableDictionary, ReadOnlyMemory> } } - static public IImmutableDictionary, ReadOnlyMemory> GetRepositoryFilesPartialForBranchViaLibGitSharpCheckout( - string cloneUrl, - string? branchName) - { - var tempWorkingDirectory = Filesystem.CreateRandomDirectoryInTempDirectory(); - - try - { - // https://github.com/libgit2/libgit2sharp/wiki/git-clone - Repository.Clone( - cloneUrl, tempWorkingDirectory, - new CloneOptions { Checkout = false, BranchName = branchName }); - - return - Composition.ToFlatDictionaryWithPathComparer( - Filesystem.GetAllFilesFromDirectory(tempWorkingDirectory) - .Where(predicate: c => c.path?[0] == ".git")); - } - catch (Exception e) - { - throw new Exception("Failed to clone from '" + cloneUrl + "'", e); - } - finally - { - DeleteLocalDirectoryRecursive(tempWorkingDirectory); - } - } - static public (string hash, CommitContent content) GetCommitHashAndContent(Commit commit) { return (commit.Sha, new CommitContent diff --git a/implement/elm-fullstack/Pine/ResultExtension.cs b/implement/elm-fullstack/Pine/ResultExtension.cs index e2334112..b4e90f30 100644 --- a/implement/elm-fullstack/Pine/ResultExtension.cs +++ b/implement/elm-fullstack/Pine/ResultExtension.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; namespace Pine; @@ -22,4 +24,17 @@ public static Result> ListCombine(this IRead return new Result>.Ok(okList); } + + static public Result, OkT> FirstOkOrErrors( + this IEnumerable>> candidates) => + candidates.Aggregate( + seed: Result, OkT>.err(ImmutableList.Empty), + func: (accumulate, candidateFunc) => + accumulate.Unpack( + fromErr: previousErrors => + candidateFunc() + .Unpack( + fromErr: newErr => Result, OkT>.err(previousErrors.Add(newErr)), + fromOk: success => Result, OkT>.ok(success)), + fromOk: _ => accumulate)); } diff --git a/implement/elm-fullstack/Program.cs b/implement/elm-fullstack/Program.cs index c314252c..a5c0de75 100644 --- a/implement/elm-fullstack/Program.cs +++ b/implement/elm-fullstack/Program.cs @@ -15,7 +15,7 @@ namespace ElmFullstack; public class Program { - static public string AppVersionId => "2022-09-29"; + static public string AppVersionId => "2022-10-05"; static int AdminInterfaceDefaultPort => 4000; diff --git a/implement/elm-fullstack/elm-fullstack.csproj b/implement/elm-fullstack/elm-fullstack.csproj index d2715388..c4e7f9c8 100644 --- a/implement/elm-fullstack/elm-fullstack.csproj +++ b/implement/elm-fullstack/elm-fullstack.csproj @@ -5,8 +5,8 @@ net7.0 ElmFullstack elm-fs - 2022.0929.0.0 - 2022.0929.0.0 + 2022.1005.0.0 + 2022.1005.0.0 enable diff --git a/implement/test-elm-fullstack/TestLoadFromGithub.cs b/implement/test-elm-fullstack/TestLoadFromGithub.cs index 54416a02..f7b61292 100644 --- a/implement/test-elm-fullstack/TestLoadFromGithub.cs +++ b/implement/test-elm-fullstack/TestLoadFromGithub.cs @@ -203,7 +203,7 @@ IImmutableDictionary, ReadOnlyMemory> consultServer( Pine.LoadFromGitHubOrGitLab.LoadFromUrl( sourceUrl: "https://github.com/elm-fullstack/elm-fullstack/blob/30c482748f531899aac2b2d4895e5f0e52258be7/README.md", getRepositoryFilesPartialForCommit: - consultServer) + request => Pine.Result, ReadOnlyMemory>>.ok(consultServer(request))) .Extract(error => throw new Exception("Failed to load from GitHub: " + error)); var blobContent = @@ -225,7 +225,7 @@ IImmutableDictionary, ReadOnlyMemory> consultServer( Pine.LoadFromGitHubOrGitLab.LoadFromUrl( sourceUrl: "https://github.com/elm-fullstack/elm-fullstack/blob/30c482748f531899aac2b2d4895e5f0e52258be7/azure-pipelines.yml", getRepositoryFilesPartialForCommit: - consultServer) + request => Pine.Result, ReadOnlyMemory>>.ok(consultServer(request))) .Extract(error => throw new Exception("Failed to load from GitHub: " + error)); var blobContent =