From 89b3189628f623fdde40667868ff84b2685f6150 Mon Sep 17 00:00:00 2001 From: Mehrshad Date: Thu, 9 Nov 2023 13:55:41 +0330 Subject: [PATCH] sanitycheck.fsx: split SanityCheckNugetPackages The conventions repository provides NugetPackagesSanityCheck. --- scripts/sanitycheck.fsx | 352 +++++----------------------------------- 1 file changed, 39 insertions(+), 313 deletions(-) diff --git a/scripts/sanitycheck.fsx b/scripts/sanitycheck.fsx index 6d83643af..a00ad9b71 100755 --- a/scripts/sanitycheck.fsx +++ b/scripts/sanitycheck.fsx @@ -115,327 +115,53 @@ let FindOffendingPrintfUsage () = let SanityCheckNugetPackages () = + let conventionDirectory = + Path.Combine(FsxHelper.RootDir.FullName, "..", "conventions") + + let cloningResult = + Process.Execute( + { Command = "git" + Arguments = + sprintf + "clone -b SanityCheckStepSquashed https://github.com/Mersho/conventions.git %s" + conventionDirectory }, + Echo.Off + ) - let notPackagesFolder (dir: DirectoryInfo): bool = - dir.FullName <> FsxHelper.NugetSolutionPackagesDir.FullName - - let notSubmodule (dir: DirectoryInfo): bool = - let getSubmoduleDirsForThisRepo (): seq = - let regex = Regex("path\s*=\s*([^\s]+)") - seq { - for regexMatch in regex.Matches (File.ReadAllText (".gitmodules")) do - let submoduleFolderRelativePath = regexMatch.Groups.[1].ToString () - let submoduleFolder = - DirectoryInfo ( - Path.Combine (Directory.GetCurrentDirectory (), submoduleFolderRelativePath) - ) - yield submoduleFolder - } - not (getSubmoduleDirsForThisRepo().Any (fun d -> dir.FullName = d.FullName)) - - // this seems to be a bug in Microsoft.Build nuget library, FIXME: report - let normalizeDirSeparatorsPaths (path: string): string = - path - .Replace('\\', Path.DirectorySeparatorChar) - .Replace('/', Path.DirectorySeparatorChar) - - let sanityCheckNugetPackagesFromSolution (sol: FileInfo) = -#if !LEGACY_FRAMEWORK - let rec findProjectFiles (): seq = - let parsedSolution = SolutionFile.Parse sol.FullName - seq { - for projPath in (parsedSolution.ProjectsInOrder.Select(fun proj -> normalizeDirSeparatorsPaths proj.AbsolutePath).ToList()) do - if projPath.ToLower().EndsWith ".fsproj" || projPath.ToLower().EndsWith ".csproj" then - yield (FileInfo projPath) - } -#else - let findPackagesDotConfigFiles (): seq = - let parsedSolution = SolutionFile.Parse sol.FullName - seq { - for projPath in (parsedSolution.ProjectsInOrder.Select(fun proj -> normalizeDirSeparatorsPaths proj.AbsolutePath).ToList()) do - if projPath.ToLower().EndsWith ".fsproj" then - for file in ((FileInfo projPath).Directory).EnumerateFiles () do - if file.Name.ToLower () = "packages.config" then - yield file - } -#endif - - let rec findNuspecFiles (dir: DirectoryInfo): seq = - dir.Refresh () - seq { - for file in dir.EnumerateFiles () do - if (file.Name.ToLower ()).EndsWith ".nuspec" then - yield file - for subdir in dir.EnumerateDirectories().Where(notSubmodule).Where(notPackagesFolder) do - for file in findNuspecFiles subdir do - yield file - } - - let getPackageTree (sol: FileInfo): Map> = -#if !LEGACY_FRAMEWORK - let projectFiles = findProjectFiles() - let projectElements = - seq { - for projectFile in projectFiles do - let xmlDoc = XDocument.Load projectFile.FullName - let query = "//PackageReference" - let pkgReferences = xmlDoc.XPathSelectElements query - - for pkgReference in pkgReferences do - let id = pkgReference.Attributes().Single(fun attr -> attr.Name.LocalName = "Include" || attr.Name.LocalName = "Update").Value - let version = pkgReference.Attributes().Single(fun attr -> attr.Name.LocalName = "Version").Value - yield { File = projectFile }, { PackageId = id; PackageVersion = version; ReqReinstall = None } - } |> List.ofSeq -#else - let packagesConfigFiles = findPackagesDotConfigFiles() - let projectElements = - seq { - for packagesConfigFile in packagesConfigFiles do - let xmlDoc = XDocument.Load packagesConfigFile.FullName - for descendant in xmlDoc.Descendants () do - if descendant.Name.LocalName.ToLower() = "package" then - let id = descendant.Attributes().Single(fun attr -> attr.Name.LocalName = "id").Value - let version = descendant.Attributes().Single(fun attr -> attr.Name.LocalName = "version").Value - let reqReinstall = descendant.Attributes().Any(fun attr -> attr.Name.LocalName = "requireReinstallation") - yield { File = packagesConfigFile }, { PackageId = id; PackageVersion = version; ReqReinstall = Some reqReinstall } - } |> List.ofSeq -#endif - - let solDir = sol.Directory - solDir.Refresh () - let nuspecFiles = findNuspecFiles solDir - let nuspecFileElements = - seq { - for nuspecFile in nuspecFiles do - let xmlDoc = XDocument.Load nuspecFile.FullName - - let nsOpt = - let nsString = xmlDoc.Root.Name.Namespace.ToString() - if String.IsNullOrEmpty nsString then - None - else - let nsManager = XmlNamespaceManager(NameTable()) - let nsPrefix = "x" - nsManager.AddNamespace(nsPrefix, nsString) - if nsString <> "http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd" then - Console.Error.WriteLine "Warning: the namespace URL doesn't match expectations, nuspec's XPath query may result in no elements" - Some(nsManager, sprintf "%s:" nsPrefix) - let query = "//{0}dependency" - let dependencies = - match nsOpt with - | None -> - let fixedQuery = String.Format(query, String.Empty) - xmlDoc.XPathSelectElements fixedQuery - | Some (nsManager, nsPrefix) -> - let fixedQuery = String.Format(query, nsPrefix) - xmlDoc.XPathSelectElements(fixedQuery, nsManager) - - for dependency in dependencies do - let id = dependency.Attributes().Single(fun attr -> attr.Name.LocalName = "id").Value - let version = dependency.Attributes().Single(fun attr -> attr.Name.LocalName = "version").Value - yield { File = nuspecFile }, { PackageId = id; PackageVersion = version; ReqReinstall = None } - } |> List.ofSeq - - let allElements = Seq.append projectElements nuspecFileElements - - allElements - |> MapHelper.MergeIntoMap - - let getAllPackageIdsAndVersions (packageTree: Map>): Map> = - seq { - for KeyValue (dependencyHolderFile, pkgs) in packageTree do - for pkg in pkgs do - yield pkg, dependencyHolderFile.DependencyHolderName - } |> MapHelper.MergeIntoMap - - let getDirectoryNamesForPackagesSet (packages: Map>): Map> = - seq { - for KeyValue (package, prjs) in packages do -#if !LEGACY_FRAMEWORK - let dirForPackage = - sprintf "%s%s%s" - (package.PackageId.ToLower()) - (Path.DirectorySeparatorChar.ToString()) - package.PackageVersion -#else - let dirForPackage = sprintf "%s.%s" package.PackageId package.PackageVersion -#endif - yield dirForPackage, prjs - } |> Map.ofSeq - - let findMissingPackageDirs (solDir: DirectoryInfo) (idealPackageDirs: Map>): Map> = - solDir.Refresh () - if not FsxHelper.NugetSolutionPackagesDir.Exists then - failwithf "'%s' subdir under solution dir %s doesn't exist, run `make` first" - FsxHelper.NugetSolutionPackagesDir.Name - FsxHelper.NugetSolutionPackagesDir.FullName - let packageDirsAbsolutePaths = FsxHelper.NugetSolutionPackagesDir.EnumerateDirectories().Select (fun dir -> dir.FullName) - if not (packageDirsAbsolutePaths.Any()) then - Console.Error.WriteLine ( - sprintf "'%s' subdir under solution dir %s doesn't contain any packages" - FsxHelper.NugetSolutionPackagesDir.Name - FsxHelper.NugetSolutionPackagesDir.FullName - ) - Console.Error.WriteLine "Maybe you forgot to issue the commands `git submodule sync --recursive && git submodule update --init --recursive`?" - Environment.Exit 1 - - seq { - for KeyValue (packageDirNameThatShouldExist, prjs) in idealPackageDirs do - let pkgDirToLookFor = - Path.Combine(FsxHelper.NugetSolutionPackagesDir.FullName, packageDirNameThatShouldExist) - |> DirectoryInfo - if not pkgDirToLookFor.Exists then - yield packageDirNameThatShouldExist, prjs - } |> Map.ofSeq - - let findExcessPackageDirs (solDir: DirectoryInfo) (idealPackageDirs: Map>): seq = - solDir.Refresh () - if not (FsxHelper.NugetSolutionPackagesDir.Exists) then - failwithf "'%s' subdir under solution dir %s doesn't exist, run `make` first" - FsxHelper.NugetSolutionPackagesDir.Name - FsxHelper.NugetSolutionPackagesDir.FullName - // "src" is a directory for source codes and build scripts, - // not for packages, so we need to exclude it from here - let packageDirNames = FsxHelper.NugetSolutionPackagesDir.EnumerateDirectories().Select(fun dir -> dir.Name).Except(["src"]) - if not (packageDirNames.Any()) then - failwithf "'%s' subdir under solution dir %s doesn't contain any packages" - FsxHelper.NugetSolutionPackagesDir.Name - FsxHelper.NugetSolutionPackagesDir.FullName - let packageDirsThatShouldExist = MapHelper.GetKeysOfMap idealPackageDirs - seq { - for packageDirThatExists in packageDirNames do - if not (packageDirsThatShouldExist.Contains packageDirThatExists) then - yield packageDirThatExists - } - - let findPackagesWithMoreThanOneVersion - (packageTree: Map>) - : Map> = - - let getAllPackageInfos (packages: Map>) = - let pkgInfos = - seq { - for KeyValue (_, pkgs) in packages do - for pkg in pkgs do - yield pkg - } - Set pkgInfos - - let getAllPackageVersionsForPackageId (packages: seq) (packageId: string) = - seq { - for package in packages do - if package.PackageId = packageId then - yield package.PackageVersion - } |> Set - - let packageInfos = getAllPackageInfos packageTree - let packageIdsWithMoreThan1Version = - seq { - for packageId in packageInfos.Select (fun pkg -> pkg.PackageId) do - let versions = getAllPackageVersionsForPackageId packageInfos packageId - if versions.Count > 1 then - yield packageId - } - if not (packageIdsWithMoreThan1Version.Any()) then - Map.empty - else - seq { - for pkgId in packageIdsWithMoreThan1Version do - let pkgs = seq { - for KeyValue (file, packageInfos) in packageTree do - for pkg in packageInfos do - if pkg.PackageId = pkgId then - yield file, pkg - } - yield pkgId, pkgs - } |> Map.ofSeq - - let packageTree = getPackageTree sol - let packages = getAllPackageIdsAndVersions packageTree - Console.WriteLine(sprintf "%d nuget packages found for solution %s" packages.Count sol.Name) - let idealDirList = getDirectoryNamesForPackagesSet packages - - let solDir = sol.Directory - solDir.Refresh () - let missingPackageDirs = findMissingPackageDirs solDir idealDirList - if missingPackageDirs.Any () then - for KeyValue(missingPkg, depHolders) in missingPackageDirs do - let depHolderNames = String.Join(",", depHolders.Select(fun dh -> dh.Name)) - Console.Error.WriteLine (sprintf "Missing folder for nuget package in submodule: %s (referenced from %s)" missingPkg depHolderNames) - Environment.Exit 1 - -#if LEGACY_FRAMEWORK - let excessPackageDirs = findExcessPackageDirs solDir idealDirList - if excessPackageDirs.Any () then - let advice = "remove it with git filter-branch to avoid needless bandwidth: http://stackoverflow.com/a/17824718/6503091" - for excessPkg in excessPackageDirs do - Console.Error.WriteLine(sprintf "Unused nuget package folder for solution dir %s: %s (%s)" solDir.Name excessPkg advice) - Environment.Exit 1 -#endif + match cloningResult.Result with + | Error _ -> + Console.WriteLine() + Console.Error.WriteLine "Clone failed." + Environment.Exit 1 - let pkgWithMoreThan1VersionPrint (key: string) (packageInfos: seq) = - Console.Error.WriteLine (sprintf "Package found with more than one version: %s. All occurrences:" key) - for file,pkgInfo in packageInfos do - Console.Error.WriteLine (sprintf "* Version: %s. Dependency holder: %s" pkgInfo.PackageVersion file.DependencyHolderName.Name) - let packagesWithMoreThanOneVersion = findPackagesWithMoreThanOneVersion packageTree - if packagesWithMoreThanOneVersion.Any() then - Map.iter pkgWithMoreThan1VersionPrint packagesWithMoreThanOneVersion - Environment.Exit 1 + | WarningsOrAmbiguous _ -> () - let findPackagesWithSomeReqReinstallAttrib - (packageTree: Map>) - : seq = - seq { - for KeyValue (file, packageInfos) in packageTree do - for pkg in packageInfos do - match pkg.ReqReinstall with - | Some true -> - yield file, pkg - | _ -> () - } - let packagesWithWithSomeReqReinstallAttrib = findPackagesWithSomeReqReinstallAttrib packageTree - if packagesWithWithSomeReqReinstallAttrib.Any() then - Console.Error.WriteLine ( - sprintf "Packages found with some RequireReinstall attribute (please reinstall it before pushing):" - ) - for file,pkg in packagesWithWithSomeReqReinstallAttrib do - Console.Error.WriteLine ( - sprintf "* Name: %s. Project: %s" pkg.PackageId file.DependencyHolderName.Name - ) - Environment.Exit 1 + | _ -> () - Console.WriteLine (sprintf "Nuget sanity check succeeded for solution dir %s" solDir.FullName) + let sanityCheckArgs = + let sanityCheckExecuteCommand = + sprintf + "fsi %s" + (Path.Combine(FsxHelper.RootDir.FullName, conventionDirectory, "scripts", "sanityCheckNuget.fsx")) + sprintf + "%s %s" + sanityCheckExecuteCommand + (FsxHelper.GetSolution SolutionFile.Default) + .FullName - let rec findSolutions (dir: DirectoryInfo): seq = - dir.Refresh () - seq { - // FIXME: avoid returning duplicates? (in case there are 2 .sln files in the same dir...) - for file in dir.EnumerateFiles () do - if file.Name.ToLower().EndsWith ".sln" then - yield file - for subdir in dir.EnumerateDirectories().Where notSubmodule do - for solution in findSolutions subdir do - yield solution - } - //let solutions = Directory.GetCurrentDirectory() |> DirectoryInfo |> findSolutions - //NOTE: we hardcode the solutions rather than the line above, because e.g. Linux OS can't build/restore iOS proj - let sol = - match Misc.GuessPlatform() with -#if LEGACY_FRAMEWORK - | Misc.Platform.Linux when "msbuild" = Environment.GetEnvironmentVariable "LegacyBuildTool" -> - FsxHelper.GetSolution SolutionFile.Linux -#endif - | Misc.Platform.Mac when "msbuild" = Environment.GetEnvironmentVariable "LegacyBuildTool" -> - FsxHelper.GetSolution SolutionFile.Mac - | _ -> - // TODO: have a windows solution file - FsxHelper.GetSolution SolutionFile.Default + Process + .Execute( + { Command = "dotnet" + Arguments = sanityCheckArgs }, + Echo.All + ) + .UnwrapDefault() + |> ignore - sanityCheckNugetPackagesFromSolution sol - FindOffendingPrintfUsage() +#if !LEGACY_FRAMEWORK SanityCheckNugetPackages() - +#endif