[CI] Skip lib for xpu binary unit test (pytorch#117514)

Skip .so and .a libraries under build/bin/ for test_xpu_bin in CI
Pull Request resolved: pytorch#117514
Approved by: https://github.com/malfet

.ci/pytorch/test.sh

@@ -690,9 +690,8 @@ test_xpu_bin(){
   TEST_REPORTS_DIR=$(pwd)/test/test-reports
   mkdir -p "$TEST_REPORTS_DIR"
 
-  for xpu_case in "${BUILD_BIN_DIR}"/*{xpu,sycl}*
-  do
-    if [[ "$xpu_case" != *"*"* ]]; then
+  for xpu_case in "${BUILD_BIN_DIR}"/*{xpu,sycl}*; do
+    if [[ "$xpu_case" != *"*"* && "$xpu_case" != *.so && "$xpu_case" != *.a ]]; then
       case_name=$(basename "$xpu_case")
       echo "Testing ${case_name} ..."
       "$xpu_case" --gtest_output=xml:"$TEST_REPORTS_DIR"/"$case_name".xml