Flink1.19.1 S3（minio）保存检查点报错

2024-09-19 17:54:02,076 INFO org.apache.flink.runtime.checkpoint.CheckpointCoordinator [] - Triggering checkpoint 1 (type=CheckpointType{name='Checkpoint', sharingFilesStrategy=FORWARD_BACKWARD}) @ 1726739642067 for job 9b67ce4decddea59ffdc6d90a8915f46.
2024-09-19 18:01:52,853 INFO org.apache.flink.runtime.state.filesystem.FsCheckpointMetadataOutputStream [] - Cannot create recoverable writer due to This s3 file system implementation does not support recoverable writers., will use the ordinary writer.
2024-09-19 18:01:52,892 WARN com.amazonaws.services.s3.internal.Mimetypes [] - Unable to find 'mime.types' file in classpath
2024-09-19 18:02:27,256 WARN org.apache.flink.runtime.checkpoint.CheckpointFailureManager [] - Failed to trigger or complete checkpoint 1 for job 9b67ce4decddea59ffdc6d90a8915f46. (0 consecutive failed attempts so far)
org.apache.flink.runtime.checkpoint.CheckpointException: Failure to finalize checkpoint.
at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.finalizeCheckpoint(CheckpointCoordinator.java:1479) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.completePendingCheckpoint(CheckpointCoordinator.java:1367) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.receiveAcknowledgeMessage(CheckpointCoordinator.java:1259) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.scheduler.ExecutionGraphHandler.lambda$acknowledgeCheckpoint$2(ExecutionGraphHandler.java:109) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.scheduler.ExecutionGraphHandler.lambda$processCheckpointCoordinatorMessage$4(ExecutionGraphHandler.java:139) ~[flink-dist-1.19.1.jar:1.19.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:840) [?:?]
Caused by: java.io.IOException: Could not flush and close the file system output stream to s3://flink/checkpoints/9b67ce4decddea59ffdc6d90a8915f46/chk-1/_metadata in order to obtain the stream state handle
at org.apache.flink.runtime.state.filesystem.FsCheckpointMetadataOutputStream.closeAndFinalizeCheckpoint(FsCheckpointMetadataOutputStream.java:149) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.state.filesystem.FsCheckpointMetadataOutputStream.closeAndFinalizeCheckpoint(FsCheckpointMetadataOutputStream.java:39) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.checkpoint.PendingCheckpoint.finalizeCheckpoint(PendingCheckpoint.java:340) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.finalizeCheckpoint(CheckpointCoordinator.java:1466) ~[flink-dist-1.19.1.jar:1.19.1]
... 7 more
Caused by: java.io.IOException: com.amazonaws.SdkClientException: Unable to execute HTTP request: Connection reset
at com.facebook.presto.hive.s3.PrestoS3FileSystem$PrestoS3OutputStream.uploadObject(PrestoS3FileSystem.java:1284) ~[?:?]
at com.facebook.presto.hive.s3.PrestoS3FileSystem$PrestoS3OutputStream.close(PrestoS3FileSystem.java:1232) ~[?:?]
at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[?:?]
at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[?:?]
at org.apache.flink.fs.s3presto.common.HadoopDataOutputStream.close(HadoopDataOutputStream.java:52) ~[?:?]
at org.apache.flink.runtime.state.filesystem.FSDataOutputStreamWrapper.closeForCommitAction(FSDataOutputStreamWrapper.java:52) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.state.filesystem.MetadataOutputStreamWrapper.closeForCommit(MetadataOutputStreamWrapper.java:63) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.state.filesystem.FsCheckpointMetadataOutputStream.closeAndFinalizeCheckpoint(FsCheckpointMetadataOutputStream.java:130) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.state.filesystem.FsCheckpointMetadataOutputStream.closeAndFinalizeCheckpoint(FsCheckpointMetadataOutputStream.java:39) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.checkpoint.PendingCheckpoint.finalizeCheckpoint(PendingCheckpoint.java:340) ~[flink-dist-1.19.1.jar:1.19.1]
at org.apache.flink.runtime.checkpoint.CheckpointCoordinator.finalizeCheckpoint(CheckpointCoordinator.java:1466) ~[flink-dist-1.19.1.jar:1.19.1]
... 7 more
Caused by: com.amazonaws.SdkClientException: Unable to execute HTTP request: Connection reset
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleRetryableException(AmazonHttpClient.java:1219) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1165) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:814) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:781) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:755) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:715) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:697) ~[?:?]
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:561) ~[?:?]
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:541) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5456) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5403) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client.access$300(AmazonS3Client.java:421) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client$PutObjectStrategy.invokeServiceCall(AmazonS3Client.java:6532) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client.uploadObject(AmazonS3Client.java:1861) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1821) ~[?:?]
at com.amazonaws.services.s3.transfer.internal.UploadCallable.uploadInOneChunk(UploadCallable.java:169) ~[?:?]
at com.amazonaws.services.s3.transfer.internal.UploadCallable.call(UploadCallable.java:149) ~[?:?]
at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:115) ~[?:?]
at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:45) ~[?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
... 3 more
Caused by: java.net.SocketException: Connection reset
at sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:328) ~[?:?]
at sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:355) ~[?:?]
at sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:808) ~[?:?]
at java.net.Socket$SocketInputStream.read(Socket.java:966) ~[?:?]
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at com.amazonaws.http.protocol.SdkHttpRequestExecutor.doReceiveResponse(SdkHttpRequestExecutor.java:82) ~[?:?]
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) ~[flink-doris-connector-1.19-24.0.0.jar:24.0.0]
at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1346) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1157) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:814) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:781) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:755) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:715) ~[?:?]
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:697) ~[?:?]
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:561) ~[?:?]
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:541) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5456) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5403) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client.access$300(AmazonS3Client.java:421) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client$PutObjectStrategy.invokeServiceCall(AmazonS3Client.java:6532) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client.uploadObject(AmazonS3Client.java:1861) ~[?:?]
at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1821) ~[?:?]
at com.amazonaws.services.s3.transfer.internal.UploadCallable.uploadInOneChunk(UploadCallable.java:169) ~[?:?]
at com.amazonaws.services.s3.transfer.internal.UploadCallable.call(UploadCallable.java:149) ~[?:?]
at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:115) ~[?:?]
at com.amazonaws.services.s3.transfer.internal.UploadMonitor.call(UploadMonitor.java:45) ~[?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
... 3 more