8254309: appcds GCDuringDump.java failed - class must exist

Reviewed-by: ccheung, iklam

Author: yminqi

src/hotspot/share/classfile/classListParser.cpp

@@ -37,6 +37,7 @@
 #include "interpreter/linkResolver.hpp"
 #include "logging/log.hpp"
 #include "logging/logTag.hpp"
+#include "memory/archiveUtils.hpp"
 #include "memory/metaspaceShared.hpp"
 #include "memory/resourceArea.hpp"
 #include "oops/constantPool.hpp"
@@ -572,6 +573,7 @@ Klass* ClassListParser::load_current_class(TRAPS) {
       klass = java_lang_Class::as_Klass(obj);
     } else { // load classes in bootclasspath/a
       if (HAS_PENDING_EXCEPTION) {
+        ArchiveUtils::check_for_oom(PENDING_EXCEPTION); // exit on OOM
         CLEAR_PENDING_EXCEPTION;
       }
 
@@ -582,6 +584,8 @@ Klass* ClassListParser::load_current_class(TRAPS) {
         } else {
           if (!HAS_PENDING_EXCEPTION) {
             THROW_NULL(vmSymbols::java_lang_ClassNotFoundException());
+          } else {
+            ArchiveUtils::check_for_oom(PENDING_EXCEPTION); // exit on OOM
           }
         }
       }
@@ -590,6 +594,9 @@ Klass* ClassListParser::load_current_class(TRAPS) {
     // If "source:" tag is specified, all super class and super interfaces must be specified in the
     // class list file.
     klass = load_class_from_source(class_name_symbol, CHECK_NULL);
+    if (HAS_PENDING_EXCEPTION) {
+      ArchiveUtils::check_for_oom(PENDING_EXCEPTION); // exit on OOM
+    }
   }
 
   if (klass != NULL && klass->is_instance_klass() && is_id_specified()) {

src/hotspot/share/memory/archiveUtils.cpp

@@ -319,3 +319,12 @@ void ArchiveUtils::log_to_classlist(BootstrapInfo* bootstrap_specifier, TRAPS) {
     }
   }
 }
+
+void ArchiveUtils::check_for_oom(oop exception) {
+  assert(exception != nullptr, "Sanity check");
+  if (exception->is_a(SystemDictionary::OutOfMemoryError_klass())) {
+    vm_exit_during_cds_dumping(
+      err_msg("Out of memory. Please run with a larger Java heap, current MaxHeapSize = " SIZE_FORMAT "M",
+              MaxHeapSize/M));
+  }
+}

src/hotspot/share/memory/archiveUtils.hpp

@@ -243,6 +243,7 @@ class ReadClosure : public SerializeClosure {
 class ArchiveUtils {
 public:
   static void log_to_classlist(BootstrapInfo* bootstrap_specifier, TRAPS) NOT_CDS_RETURN;
+  static void check_for_oom(oop exception) NOT_CDS_RETURN;
 };
 
 #endif // SHARE_MEMORY_ARCHIVEUTILS_HPP

src/hotspot/share/memory/heapShared.cpp

@@ -259,15 +259,6 @@ void HeapShared::run_full_gc_in_vm_thread() {
 
 void HeapShared::archive_java_heap_objects(GrowableArray<MemRegion> *closed,
                                            GrowableArray<MemRegion> *open) {
-  if (!is_heap_object_archiving_allowed()) {
-    log_info(cds)(
-      "Archived java heap is not supported as UseG1GC, "
-      "UseCompressedOops and UseCompressedClassPointers are required."
-      "Current settings: UseG1GC=%s, UseCompressedOops=%s, UseCompressedClassPointers=%s.",
-      BOOL_TO_STR(UseG1GC), BOOL_TO_STR(UseCompressedOops),
-      BOOL_TO_STR(UseCompressedClassPointers));
-    return;
-  }
 
   G1HeapVerifier::verify_ready_for_archiving();
 
@@ -1035,7 +1026,13 @@ void HeapShared::init_subgraph_entry_fields(ArchivableStaticFieldInfo fields[],
     TempNewSymbol field_name =  SymbolTable::new_symbol(info->field_name);
 
     Klass* k = SystemDictionary::resolve_or_null(klass_name, THREAD);
-    assert(k != NULL && !HAS_PENDING_EXCEPTION, "class must exist");
+    if (HAS_PENDING_EXCEPTION) {
+      ResourceMark rm(THREAD);
+      ArchiveUtils::check_for_oom(PENDING_EXCEPTION); // exit on OOM
+      log_info(cds)("%s: %s", PENDING_EXCEPTION->klass()->external_name(),
+                    java_lang_String::as_utf8_string(java_lang_Throwable::message(PENDING_EXCEPTION)));
+      vm_exit_during_initialization("VM exits due to exception, use -Xlog:cds,exceptions=trace for detail");
+    }
     InstanceKlass* ik = InstanceKlass::cast(k);
     assert(InstanceKlass::cast(ik)->is_shared_boot_class(),
            "Only support boot classes");
@@ -1052,8 +1049,8 @@ void HeapShared::init_subgraph_entry_fields(ArchivableStaticFieldInfo fields[],
 }
 
 void HeapShared::init_subgraph_entry_fields(Thread* THREAD) {
+  assert(is_heap_object_archiving_allowed(), "Sanity check");
   _dump_time_subgraph_info_table = new (ResourceObj::C_HEAP, mtClass)DumpTimeKlassSubGraphInfoTable();
-
   init_subgraph_entry_fields(closed_archive_subgraph_entry_fields,
                              num_closed_archive_subgraph_entry_fields,
                              THREAD);
@@ -1068,8 +1065,10 @@ void HeapShared::init_subgraph_entry_fields(Thread* THREAD) {
 }
 
 void HeapShared::init_for_dumping(Thread* THREAD) {
-  _dumped_interned_strings = new (ResourceObj::C_HEAP, mtClass)DumpedInternedStrings();
-  init_subgraph_entry_fields(THREAD);
+  if (is_heap_object_archiving_allowed()) {
+    _dumped_interned_strings = new (ResourceObj::C_HEAP, mtClass)DumpedInternedStrings();
+    init_subgraph_entry_fields(THREAD);
+  }
 }
 
 void HeapShared::archive_object_subgraphs(ArchivableStaticFieldInfo fields[],

src/hotspot/share/memory/metaspaceShared.cpp

@@ -1166,6 +1166,15 @@ bool MetaspaceShared::try_link_class(InstanceKlass* ik, TRAPS) {
 
 #if INCLUDE_CDS_JAVA_HEAP
 void VM_PopulateDumpSharedSpace::dump_java_heap_objects() {
+  if(!HeapShared::is_heap_object_archiving_allowed()) {
+    log_info(cds)(
+      "Archived java heap is not supported as UseG1GC, "
+      "UseCompressedOops and UseCompressedClassPointers are required."
+      "Current settings: UseG1GC=%s, UseCompressedOops=%s, UseCompressedClassPointers=%s.",
+      BOOL_TO_STR(UseG1GC), BOOL_TO_STR(UseCompressedOops),
+      BOOL_TO_STR(UseCompressedClassPointers));
+    return;
+  }
   // Find all the interned strings that should be dumped.
   int i;
   for (i = 0; i < _global_klass_objects->length(); i++) {

test/hotspot/jtreg/TEST.groups

@@ -325,6 +325,7 @@ hotspot_appcds_dynamic = \
  -runtime/cds/appcds/dynamicArchive \
  -runtime/cds/appcds/loaderConstraints/DynamicLoaderConstraintsTest.java \
  -runtime/cds/appcds/javaldr/ArrayTest.java \
+ -runtime/cds/appcds/javaldr/ExceptionDuringDumpAtObjectsInitPhase.java \
  -runtime/cds/appcds/javaldr/GCSharedStringsDuringDump.java \
  -runtime/cds/appcds/javaldr/HumongousDuringDump.java \
  -runtime/cds/appcds/javaldr/LockDuringDump.java \

test/hotspot/jtreg/runtime/cds/appcds/javaldr/ExceptionDuringDumpAtObjectsInitPhase.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @summary Out of memory When dumping the CDS archive
+ * @library /test/lib /test/hotspot/jtreg/runtime/cds/appcds /test/hotspot/jtreg/runtime/cds/appcds/test-classes
+ * @requires vm.cds.archived.java.heap
+ * @requires vm.jvmti
+ * @run driver ExceptionDuringDumpAtObjectsInitPhase
+ */
+
+import jdk.test.lib.cds.CDSOptions;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+public class ExceptionDuringDumpAtObjectsInitPhase {
+    public static String appClasses[] = {
+        Hello.class.getName(),
+    };
+    public static String agentClasses[] = {
+        GCDuringDumpTransformer.class.getName(),
+        GCDuringDumpTransformer.MyCleaner.class.getName(),
+    };
+
+    public static void main(String[] args) throws Throwable {
+        String agentJar =
+            ClassFileInstaller.writeJar("GCDuringDumpTransformer.jar",
+                                        ClassFileInstaller.Manifest.fromSourceFile("GCDuringDumpTransformer.mf"),
+                                        agentClasses);
+
+        String appJar =
+            ClassFileInstaller.writeJar("GCDuringDumpApp.jar", appClasses);
+
+        String gcLog = Boolean.getBoolean("test.cds.verbose.gc") ?
+            "-Xlog:gc*=info,gc+region=trace,gc+alloc+region=debug" : "-showversion";
+
+        // 1. Test with exception
+        System.out.println("1. Exception during dump");
+        TestCommon.dump(appJar,
+                        TestCommon.list(Hello.class.getName()),
+                        "-XX:+UnlockDiagnosticVMOptions",
+                        "-XX:+AllowArchivingWithJavaAgent",
+                        "-javaagent:" + agentJar,
+                        "-Xlog:cds,class+load",
+                        "-Xmx32m",
+                        "-Dtest.with.exception=true",
+                        gcLog).shouldNotHaveExitValue(0)
+                              .shouldContain("Preload Error: Failed to load jdk/internal/math/FDBigInteger")
+                              .shouldContain("VM exits due to exception, use -Xlog:cds,exceptions=trace for detail");
+
+        // 2. Test with OOM
+        System.out.println("2. OOM during dump");
+        TestCommon.dump(appJar,
+                        TestCommon.list(Hello.class.getName()),
+                        "-XX:+UnlockDiagnosticVMOptions",
+                        "-XX:+AllowArchivingWithJavaAgent",
+                        "-javaagent:" + agentJar,
+                        "-Dtest.with.oom=true",
+                        "-Xlog:cds,class+load",
+                        "-Xmx12M",
+                        gcLog).shouldNotHaveExitValue(0)
+                              .shouldContain("Out of memory. Please run with a larger Java heap, current MaxHeapSize");
+    }
+}

test/hotspot/jtreg/runtime/cds/appcds/javaldr/GCDuringDumpTransformer.java

@@ -26,10 +26,16 @@
 import java.lang.instrument.Instrumentation;
 import java.lang.instrument.IllegalClassFormatException;
 import java.lang.ref.Cleaner;
+import java.util.ArrayList;
+import java.util.List;
 import java.security.ProtectionDomain;
 
 public class GCDuringDumpTransformer implements ClassFileTransformer {
     static boolean TEST_WITH_CLEANER = Boolean.getBoolean("test.with.cleaner");
+    static boolean TEST_WITH_EXCEPTION = Boolean.getBoolean("test.with.exception");
+    static boolean TEST_WITH_OOM = Boolean.getBoolean("test.with.oom");
+    static List<byte[]> waste = new ArrayList();
+
     static Cleaner cleaner;
     static Thread thread;
     static Object garbage;
@@ -45,6 +51,22 @@ public class GCDuringDumpTransformer implements ClassFileTransformer {
 
     public byte[] transform(ClassLoader loader, String name, Class<?> classBeingRedefined,
                             ProtectionDomain pd, byte[] buffer) throws IllegalClassFormatException {
+        // jdk/internal/math/FDBigInteger is loaded as part of archived heap.
+        if (name.equals("jdk/internal/math/FDBigInteger")) {
+            System.out.println("Transforming class jdk/internal/math/FDBigInteger");
+            if (TEST_WITH_EXCEPTION) {
+              System.out.println("Return bad buffer for " + name);
+              return new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
+            }
+            if (TEST_WITH_OOM) {
+                // fill until OOM
+                System.out.println("Fill objects until OOM");
+                for (;;) {
+                    waste.add(new byte[64*1024]);
+                }
+            }
+        }
+
         if (TEST_WITH_CLEANER) {
             if (name.equals("Hello")) {
                 garbage = null;
@@ -60,14 +82,15 @@ public byte[] transform(ClassLoader loader, String name, Class<?> classBeingRede
                 } catch (Throwable t2) {}
             }
         }
-
         return null;
     }
 
     private static Instrumentation savedInstrumentation;
 
     public static void premain(String agentArguments, Instrumentation instrumentation) {
         System.out.println("ClassFileTransformer.premain() is called: TEST_WITH_CLEANER = " + TEST_WITH_CLEANER);
+        System.out.println("ClassFileTransformer.premain() is called: TEST_WITH_EXCEPTION = " + TEST_WITH_EXCEPTION);
+        System.out.println("ClassFileTransformer.premain() is called: TEST_WITH_OOM = " + TEST_WITH_OOM);
         instrumentation.addTransformer(new GCDuringDumpTransformer(), /*canRetransform=*/true);
         savedInstrumentation = instrumentation;
     }