Merge branch 'js/ci-windows-update' into maint

Updates to the CI settings.

* js/ci-windows-update:
  Azure Pipeline: switch to the latest agent pools
  ci: prevent `perforce` from being quarantined
  t/lib-httpd: avoid using macOS' sed

azure-pipelines.yml

@@ -5,7 +5,8 @@ jobs:
 - job: windows_build
   displayName: Windows Build
   condition: succeeded()
-  pool: Hosted
+  pool:
+    vmImage: windows-latest
   timeoutInMinutes: 240
   steps:
   - powershell: |
@@ -61,7 +62,8 @@ jobs:
   displayName: Windows Test
   dependsOn: windows_build
   condition: succeeded()
-  pool: Hosted
+  pool:
+    vmImage: windows-latest
   timeoutInMinutes: 240
   strategy:
     parallel: 10
@@ -133,7 +135,8 @@ jobs:
 - job: vs_build
   displayName: Visual Studio Build
   condition: succeeded()
-  pool: Hosted VS2017
+  pool:
+    vmImage: windows-latest
   timeoutInMinutes: 240
   steps:
   - powershell: |
@@ -181,6 +184,7 @@ jobs:
       platform: x64
       configuration: Release
       maximumCpuCount: 4
+      msbuildArguments: /p:PlatformToolset=v142
   - powershell: |
       & compat\vcbuild\vcpkg_copy_dlls.bat release
       if (!$?) { exit(1) }
@@ -224,7 +228,8 @@ jobs:
   displayName: Visual Studio Test
   dependsOn: vs_build
   condition: succeeded()
-  pool: Hosted
+  pool:
+    vmImage: windows-latest
   timeoutInMinutes: 240
   strategy:
     parallel: 10
@@ -292,7 +297,8 @@ jobs:
 - job: linux_clang
   displayName: linux-clang
   condition: succeeded()
-  pool: Hosted Ubuntu 1604
+  pool:
+    vmImage: ubuntu-latest
   steps:
   - bash: |
        test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1
@@ -330,7 +336,8 @@ jobs:
 - job: linux_gcc
   displayName: linux-gcc
   condition: succeeded()
-  pool: Hosted Ubuntu 1604
+  pool:
+    vmImage: ubuntu-latest
   steps:
   - bash: |
        test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1
@@ -367,7 +374,8 @@ jobs:
 - job: osx_clang
   displayName: osx-clang
   condition: succeeded()
-  pool: Hosted macOS
+  pool:
+    vmImage: macOS-latest
   steps:
   - bash: |
        test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1
@@ -402,7 +410,8 @@ jobs:
 - job: osx_gcc
   displayName: osx-gcc
   condition: succeeded()
-  pool: Hosted macOS
+  pool:
+    vmImage: macOS-latest
   steps:
   - bash: |
        test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1
@@ -435,7 +444,8 @@ jobs:
 - job: gettext_poison
   displayName: GETTEXT_POISON
   condition: succeeded()
-  pool: Hosted Ubuntu 1604
+  pool:
+    vmImage: ubuntu-latest
   steps:
   - bash: |
        test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1
@@ -472,7 +482,8 @@ jobs:
 - job: linux32
   displayName: Linux32
   condition: succeeded()
-  pool: Hosted Ubuntu 1604
+  pool:
+    vmImage: ubuntu-latest
   steps:
   - bash: |
        test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1
@@ -506,7 +517,8 @@ jobs:
 - job: static_analysis
   displayName: StaticAnalysis
   condition: succeeded()
-  pool: Hosted Ubuntu 1604
+  pool:
+    vmImage: ubuntu-latest
   steps:
   - bash: |
        test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1
@@ -526,7 +538,8 @@ jobs:
 - job: documentation
   displayName: Documentation
   condition: succeeded()
-  pool: Hosted Ubuntu 1604
+  pool:
+    vmImage: ubuntu-latest
   steps:
   - bash: |
        test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1

ci/install-dependencies.sh

@@ -40,11 +40,11 @@ osx-clang|osx-gcc)
 	test -z "$BREW_INSTALL_PACKAGES" ||
 	brew install $BREW_INSTALL_PACKAGES
 	brew link --force gettext
-	brew cask install perforce || {
+	brew cask install --no-quarantine perforce || {
 		# Update the definitions and try again
 		cask_repo="$(brew --repository)"/Library/Taps/homebrew/homebrew-cask &&
 		git -C "$cask_repo" pull --no-stat &&
-		brew cask install perforce
+		brew cask install --no-quarantine perforce
 	} ||
 	brew install caskroom/cask/perforce
 	case "$jobname" in

t/lib-httpd.sh

@@ -132,7 +132,7 @@ prepare_httpd() {
 	install_script broken-smart-http.sh
 	install_script error-smart-http.sh
 	install_script error.sh
-	install_script apply-one-time-sed.sh
+	install_script apply-one-time-perl.sh
 
 	ln -s "$LIB_HTTPD_MODULE_PATH" "$HTTPD_ROOT_PATH/modules"
 

t/lib-httpd/apache.conf

@@ -113,7 +113,7 @@ Alias /auth/dumb/ www/auth/dumb/
 	SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}
 	SetEnv GIT_HTTP_EXPORT_ALL
 </LocationMatch>
-<LocationMatch /one_time_sed/>
+<LocationMatch /one_time_perl/>
 	SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}
 	SetEnv GIT_HTTP_EXPORT_ALL
 </LocationMatch>
@@ -122,7 +122,7 @@ ScriptAliasMatch /smart_*[^/]*/(.*) ${GIT_EXEC_PATH}/git-http-backend/$1
 ScriptAlias /broken_smart/ broken-smart-http.sh/
 ScriptAlias /error_smart/ error-smart-http.sh/
 ScriptAlias /error/ error.sh/
-ScriptAliasMatch /one_time_sed/(.*) apply-one-time-sed.sh/$1
+ScriptAliasMatch /one_time_perl/(.*) apply-one-time-perl.sh/$1
 <Directory ${GIT_EXEC_PATH}>
 	Options FollowSymlinks
 </Directory>
@@ -135,7 +135,7 @@ ScriptAliasMatch /one_time_sed/(.*) apply-one-time-sed.sh/$1
 <Files error.sh>
   Options ExecCGI
 </Files>
-<Files apply-one-time-sed.sh>
+<Files apply-one-time-perl.sh>
 	Options ExecCGI
 </Files>
 <Files ${GIT_EXEC_PATH}/git-http-backend>

t/lib-httpd/apply-one-time-perl.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# If "one-time-perl" exists in $HTTPD_ROOT_PATH, run perl on the HTTP response,
+# using the contents of "one-time-perl" as the perl command to be run. If the
+# response was modified as a result, delete "one-time-perl" so that subsequent
+# HTTP responses are no longer modified.
+#
+# This can be used to simulate the effects of the repository changing in
+# between HTTP request-response pairs.
+if test -f one-time-perl
+then
+	LC_ALL=C
+	export LC_ALL
+
+	"$GIT_EXEC_PATH/git-http-backend" >out
+	perl -pe "$(cat one-time-perl)" out >out_modified
+
+	if cmp -s out out_modified
+	then
+		cat out
+	else
+		cat out_modified
+		rm one-time-perl
+	fi
+else
+	"$GIT_EXEC_PATH/git-http-backend"
+fi

t/t5537-fetch-shallow.sh

@@ -233,7 +233,7 @@ test_expect_success 'shallow fetches check connectivity before writing shallow f
 	git -C "$REPO" config protocol.version 2 &&
 	git -C client config protocol.version 2 &&
 
-	git -C client fetch --depth=2 "$HTTPD_URL/one_time_sed/repo" master:a_branch &&
+	git -C client fetch --depth=2 "$HTTPD_URL/one_time_perl/repo" master:a_branch &&
 
 	# Craft a situation in which the server sends back an unshallow request
 	# with an empty packfile. This is done by refetching with a shorter
@@ -242,13 +242,13 @@ test_expect_success 'shallow fetches check connectivity before writing shallow f
 	printf "s/0034shallow %s/0036unshallow %s/" \
 	       "$(git -C "$REPO" rev-parse HEAD)" \
 	       "$(git -C "$REPO" rev-parse HEAD^)" \
-	       >"$HTTPD_ROOT_PATH/one-time-sed" &&
+	       >"$HTTPD_ROOT_PATH/one-time-perl" &&
 	test_must_fail env GIT_TEST_SIDEBAND_ALL=0 git -C client \
-		fetch --depth=1 "$HTTPD_URL/one_time_sed/repo" \
+		fetch --depth=1 "$HTTPD_URL/one_time_perl/repo" \
 		master:a_branch &&
 
-	# Ensure that the one-time-sed script was used.
-	! test -e "$HTTPD_ROOT_PATH/one-time-sed" &&
+	# Ensure that the one-time-perl script was used.
+	! test -e "$HTTPD_ROOT_PATH/one-time-perl" &&
 
 	# Ensure that the resulting repo is consistent, despite our failure to
 	# fetch.

t/t5616-partial-clone.sh

@@ -429,14 +429,18 @@ intersperse () {
 	sed 's/\(..\)/'$1'\1/g'
 }
 
-# Create a one-time-sed command to replace the existing packfile with $1.
+# Create a one-time-perl command to replace the existing packfile with $1.
 replace_packfile () {
 	# The protocol requires that the packfile be sent in sideband 1, hence
 	# the extra \x01 byte at the beginning.
-	printf "1,/packfile/!c %04x\\\\x01%s0000" \
-		"$(($(wc -c <$1) + 5))" \
-		"$(hex_unpack <$1 | intersperse '\\x')" \
-		>"$HTTPD_ROOT_PATH/one-time-sed"
+	cp $1 "$HTTPD_ROOT_PATH/one-time-pack" &&
+	echo 'if (/packfile/) {
+		print;
+		my $length = -s "one-time-pack";
+		printf "%04x\x01", $length + 5;
+		print `cat one-time-pack` . "0000";
+		last
+	}' >"$HTTPD_ROOT_PATH/one-time-perl"
 }
 
 test_expect_success 'upon cloning, check that all refs point to objects' '
@@ -460,16 +464,16 @@ test_expect_success 'upon cloning, check that all refs point to objects' '
 	# \x01 byte at the beginning.
 	replace_packfile incomplete.pack &&
 
-	# Use protocol v2 because the sed command looks for the "packfile"
+	# Use protocol v2 because the perl command looks for the "packfile"
 	# section header.
 	test_config -C "$SERVER" protocol.version 2 &&
 	test_must_fail git -c protocol.version=2 clone \
-		--filter=blob:none $HTTPD_URL/one_time_sed/server repo 2>err &&
+		--filter=blob:none $HTTPD_URL/one_time_perl/server repo 2>err &&
 
 	test_i18ngrep "did not send all necessary objects" err &&
 
-	# Ensure that the one-time-sed script was used.
-	! test -e "$HTTPD_ROOT_PATH/one-time-sed"
+	# Ensure that the one-time-perl script was used.
+	! test -e "$HTTPD_ROOT_PATH/one-time-perl"
 '
 
 test_expect_success 'when partial cloning, tolerate server not sending target of tag' '
@@ -500,17 +504,17 @@ test_expect_success 'when partial cloning, tolerate server not sending target of
 	# \x01 byte at the beginning.
 	replace_packfile incomplete.pack &&
 
-	# Use protocol v2 because the sed command looks for the "packfile"
+	# Use protocol v2 because the perl command looks for the "packfile"
 	# section header.
 	test_config -C "$SERVER" protocol.version 2 &&
 
 	# Exercise to make sure it works.
 	git -c protocol.version=2 clone \
-		--filter=blob:none $HTTPD_URL/one_time_sed/server repo 2> err &&
+		--filter=blob:none $HTTPD_URL/one_time_perl/server repo 2> err &&
 	! grep "missing object referenced by" err &&
 
-	# Ensure that the one-time-sed script was used.
-	! test -e "$HTTPD_ROOT_PATH/one-time-sed"
+	# Ensure that the one-time-perl script was used.
+	! test -e "$HTTPD_ROOT_PATH/one-time-perl"
 '
 
 test_expect_success 'tolerate server sending REF_DELTA against missing promisor objects' '
@@ -533,7 +537,7 @@ test_expect_success 'tolerate server sending REF_DELTA against missing promisor
 
 	# Clone. The client has deltabase_have but not deltabase_missing.
 	git -c protocol.version=2 clone --no-checkout \
-		--filter=blob:none $HTTPD_URL/one_time_sed/server repo &&
+		--filter=blob:none $HTTPD_URL/one_time_perl/server repo &&
 	git -C repo hash-object -w -- "$SERVER/have.txt" &&
 
 	# Sanity check to ensure that the client does not have
@@ -574,7 +578,7 @@ test_expect_success 'tolerate server sending REF_DELTA against missing promisor
 
 	replace_packfile thin.pack &&
 
-	# Use protocol v2 because the sed command looks for the "packfile"
+	# Use protocol v2 because the perl command looks for the "packfile"
 	# section header.
 	test_config -C "$SERVER" protocol.version 2 &&
 
@@ -587,8 +591,8 @@ test_expect_success 'tolerate server sending REF_DELTA against missing promisor
 	grep "want $(cat deltabase_missing)" trace &&
 	! grep "want $(cat deltabase_have)" trace &&
 
-	# Ensure that the one-time-sed script was used.
-	! test -e "$HTTPD_ROOT_PATH/one-time-sed"
+	# Ensure that the one-time-perl script was used.
+	! test -e "$HTTPD_ROOT_PATH/one-time-perl"
 '
 
 # DO NOT add non-httpd-specific tests here, because the last part of this

t/t5702-protocol-v2.sh

@@ -712,11 +712,11 @@ test_expect_success 'when server sends "ready", expect DELIM' '
 
 	# After "ready" in the acknowledgments section, pretend that a FLUSH
 	# (0000) was sent instead of a DELIM (0001).
-	printf "/ready/,$ s/0001/0000/" \
-		>"$HTTPD_ROOT_PATH/one-time-sed" &&
+	printf "\$ready = 1 if /ready/; \$ready && s/0001/0000/" \
+		>"$HTTPD_ROOT_PATH/one-time-perl" &&
 
 	test_must_fail git -C http_child -c protocol.version=2 \
-		fetch "$HTTPD_URL/one_time_sed/http_parent" 2> err &&
+		fetch "$HTTPD_URL/one_time_perl/http_parent" 2> err &&
 	test_i18ngrep "expected packfile to be sent after .ready." err
 '
 
@@ -737,12 +737,12 @@ test_expect_success 'when server does not send "ready", expect FLUSH' '
 
 	# After the acknowledgments section, pretend that a DELIM
 	# (0001) was sent instead of a FLUSH (0000).
-	printf "/acknowledgments/,$ s/0000/0001/" \
-		>"$HTTPD_ROOT_PATH/one-time-sed" &&
+	printf "\$ack = 1 if /acknowledgments/; \$ack && s/0000/0001/" \
+		>"$HTTPD_ROOT_PATH/one-time-perl" &&
 
 	test_must_fail env GIT_TRACE_PACKET="$(pwd)/log" git -C http_child \
 		-c protocol.version=2 \
-		fetch "$HTTPD_URL/one_time_sed/http_parent" 2> err &&
+		fetch "$HTTPD_URL/one_time_perl/http_parent" 2> err &&
 	grep "fetch< .*acknowledgments" log &&
 	! grep "fetch< .*ready" log &&
 	test_i18ngrep "expected no other sections to be sent after no .ready." err