Reduce spammy logging for object store failed opens

Do not log all failed attempts. Instead, set the exception message. pr-link: Alluxio#11992 change-id: cid-63bd31bf74367098834abf49bae0f981dd725ffe
magicsilence · Aug 17, 2020 · 0756e57 · 0756e57
1 parent 278b8f5
commit 0756e57
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 22 deletions.
diff --git a/underfs/cos/src/main/java/alluxio/underfs/cos/COSInputStream.java b/underfs/cos/src/main/java/alluxio/underfs/cos/COSInputStream.java
@@ -23,11 +23,12 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.concurrent.NotThreadSafe;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
+import javax.annotation.concurrent.NotThreadSafe;
+
 /**
  * A stream for reading a file from COS. This input stream returns 0 when calling read with an empty
  * buffer.
@@ -97,21 +98,23 @@ protected InputStream createStream(long startPos, long endPos)
     // COS returns entire object if we read past the end
     req.setRange(startPos, endPos < mContentLength ? endPos - 1 : mContentLength - 1);
     CosServiceException lastException = null;
+    String errorMessage = String.format("Failed to open key: %s bucket: %s", mKey, mBucketName);
     while (mRetryPolicy.attempt()) {
       try {
         COSObject object = mCosClient.getObject(req);
         return new BufferedInputStream(object.getObjectContent());
       } catch (CosServiceException e) {
-        LOG.warn("Attempt {} to open key {} in bucket {} failed with exception : {}",
-            mRetryPolicy.getAttemptCount(), mKey, mBucketName, e.toString());
+        errorMessage = String
+            .format("Failed to open key: %s bucket: %s attempts: %d error: %s", mKey, mBucketName,
+                mRetryPolicy.getAttemptCount(), e.getMessage());
         if (e.getStatusCode() != HttpStatus.SC_NOT_FOUND) {
-          throw new IOException(e);
+          throw new IOException(errorMessage, e);
         }
         // Key does not exist
         lastException = e;
       }
     }
     // Failed after retrying key does not exist
-    throw new IOException(lastException);
+    throw new IOException(errorMessage, lastException);
   }
 }
diff --git a/underfs/gcs/src/main/java/alluxio/underfs/gcs/GCSInputStream.java b/underfs/gcs/src/main/java/alluxio/underfs/gcs/GCSInputStream.java
@@ -151,6 +151,7 @@ public long skip(long n) throws IOException {
    */
   private void openStream() throws IOException {
     ServiceException lastException = null;
+    String errorMessage = String.format("Failed to open key: %s bucket: %s", mKey, mBucketName);
     while (mRetryPolicy.attempt()) {
       try {
         GSObject object;
@@ -162,17 +163,18 @@ private void openStream() throws IOException {
         mInputStream = new BufferedInputStream(object.getDataInputStream());
         return;
       } catch (ServiceException e) {
-        LOG.warn("Attempt {} to open key {} on position {} in bucket {} failed with exception : {}",
-            mRetryPolicy.getAttemptCount(), mKey, mPos, mBucketName, e.toString());
+        errorMessage = String
+            .format("Failed to open key: %s bucket: %s attempts: %d error: %s", mKey, mBucketName,
+                mRetryPolicy.getAttemptCount(), e.getMessage());
         if (e.getResponseCode() != HttpStatus.SC_NOT_FOUND) {
-          throw new IOException(e);
+          throw new IOException(errorMessage, e);
         }
         // Key does not exist
         lastException = e;
       }
     }
     // Failed after retrying key does not exist
-    throw new IOException(lastException);
+    throw new IOException(errorMessage, lastException);
   }
 
   /**

diff --git a/underfs/kodo/src/main/java/alluxio/underfs/kodo/KodoInputStream.java b/underfs/kodo/src/main/java/alluxio/underfs/kodo/KodoInputStream.java
@@ -73,18 +73,20 @@ public class KodoInputStream extends MultiRangeObjectInputStream {
   protected InputStream createStream(long startPos, long endPos)
       throws IOException {
     IOException lastException = null;
+    String errorMessage = String.format("Failed to open key: %s", mKey);
     while (mRetryPolicy.attempt()) {
       try {
         return mKodoclent.getObject(mKey, startPos, endPos, mContentLength);
       } catch (NotFoundException e) {
-        LOG.warn("Attempt {} to open key {} failed with exception : {}",
-            mRetryPolicy.getAttemptCount(), mKey, e.toString());
+        errorMessage = String
+            .format("Failed to open key: %s attempts: %s error: %s", mKey,
+                mRetryPolicy.getAttemptCount(), e.getMessage());
         // Key does not exist
         lastException = e;
       }
     }
     // Failed after retrying key does not exist
-    throw lastException;
+    throw new IOException(errorMessage, lastException);
   }
 }
 
diff --git a/underfs/oss/src/main/java/alluxio/underfs/oss/OSSInputStream.java b/underfs/oss/src/main/java/alluxio/underfs/oss/OSSInputStream.java
@@ -15,7 +15,6 @@
 import alluxio.underfs.MultiRangeObjectInputStream;
 
 import com.aliyun.oss.OSS;
-import com.google.common.base.Throwables;
 import com.aliyun.oss.OSSException;
 import com.aliyun.oss.model.GetObjectRequest;
 import com.aliyun.oss.model.OSSObject;
@@ -98,22 +97,23 @@ protected InputStream createStream(long startPos, long endPos)
     // OSS returns entire object if we read past the end
     req.setRange(startPos, endPos < mContentLength ? endPos - 1 : mContentLength - 1);
     OSSException lastException = null;
+    String errorMessage = String.format("Failed to open key: %s bucket: %s", mKey, mBucketName);
     while (mRetryPolicy.attempt()) {
       try {
         OSSObject ossObject = mOssClient.getObject(req);
         return new BufferedInputStream(ossObject.getObjectContent());
       } catch (OSSException e) {
-        LOG.warn("Attempt {} to open key {} in bucket {} failed with exception : {}",
-            mRetryPolicy.getAttemptCount(), mKey, mBucketName, e.toString());
-        LOG.warn("OSSException " + Throwables.getStackTraceAsString(e));
+        errorMessage = String
+            .format("Failed to open key: %s bucket: %s attempts: %d error: %s", mKey, mBucketName,
+                mRetryPolicy.getAttemptCount(), e.getMessage());
         if (!e.getErrorCode().equals("NoSuchKey")) {
-          throw new IOException(e);
+          throw new IOException(errorMessage, e);
         }
         // Key does not exist
         lastException = e;
       }
     }
     // Failed after retrying key does not exist
-    throw new IOException(lastException);
+    throw new IOException(errorMessage, lastException);
   }
 }
diff --git a/underfs/s3a/src/main/java/alluxio/underfs/s3a/S3AInputStream.java b/underfs/s3a/src/main/java/alluxio/underfs/s3a/S3AInputStream.java
@@ -23,6 +23,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+
 import javax.annotation.concurrent.NotThreadSafe;
 
 /**
@@ -143,22 +144,24 @@ private void openStream() throws IOException {
       getReq.setRange(mPos);
     }
     AmazonS3Exception lastException = null;
+    String errorMessage = String.format("Failed to open key: %s bucket: %s", mKey, mBucketName);
     while (mRetryPolicy.attempt()) {
       try {
         mIn = mClient.getObject(getReq).getObjectContent();
         return;
       } catch (AmazonS3Exception e) {
-        LOG.warn("Attempt {} to open key {} in bucket {} failed with exception : {}",
-            mRetryPolicy.getAttemptCount(), mKey, mBucketName, e.toString());
+        errorMessage = String
+            .format("Failed to open key: %s bucket: %s attempts: %d error: %s", mKey, mBucketName,
+                mRetryPolicy.getAttemptCount(), e.getMessage());
         if (e.getStatusCode() != HttpStatus.SC_NOT_FOUND) {
-          throw new IOException(e);
+          throw new IOException(errorMessage, e);
         }
         // Key does not exist
         lastException = e;
       }
     }
     // Failed after retrying key does not exist
-    throw new IOException(lastException);
+    throw new IOException(errorMessage, lastException);
   }
 
   /**