8250646: hdiutil detach fix JDK-8245311 still fails sometimes

Reviewed-by: herrick, asemenyuk

Author: Alexander Matveev

src/jdk.incubator.jpackage/macosx/classes/jdk/incubator/jpackage/internal/MacDmgBundler.java

@@ -446,9 +446,22 @@ private Path buildDMG( Map<String, ? super Object> params,
             // "hdiutil detach" might not work right away due to resource busy error, so
             // repeat detach several times.
             RetryExecutor retryExecutor = new RetryExecutor();
-            // 10 times with 3 second delays.
-            retryExecutor.setMaxAttemptsCount(10).setAttemptTimeoutMillis(3000)
-                    .execute(pb);
+            // Image can get detach even if we got resource busy error, so stop
+            // trying to detach it if it is no longer attached.
+            retryExecutor.setExecutorInitializer(exec -> {
+                if (!Files.exists(mountedRoot)) {
+                    retryExecutor.abort();
+                }
+            });
+            try {
+                // 10 times with 3 second delays.
+                retryExecutor.setMaxAttemptsCount(10).setAttemptTimeoutMillis(3000)
+                        .execute(pb);
+            } catch (IOException ex) {
+                if (!retryExecutor.isAborted()) {
+                    throw ex;
+                }
+            }
         }
 
         // Compress it to a new image

src/jdk.incubator.jpackage/share/classes/jdk/incubator/jpackage/internal/RetryExecutor.java

@@ -53,6 +53,10 @@ void abort() {
         aborted = true;
     }
 
+    boolean isAborted() {
+        return aborted;
+    }
+
     static RetryExecutor retryOnKnownErrorMessage(String v) {
         RetryExecutor result = new RetryExecutor();
         return result.setExecutorInitializer(exec -> {
@@ -75,6 +79,10 @@ void execute(ProcessBuilder pb) throws IOException {
     private void executeLoop(Supplier<Executor> execSupplier) throws IOException {
         aborted = false;
         for (;;) {
+            if (aborted) {
+                break;
+            }
+
             try {
                 Executor exec = execSupplier.get();
                 if (executorInitializer != null) {