8275037: Test vmTestbase/nsk/sysdict/vm/stress/btree/btree011/btree011.java crashes with memory exhaustion on Windows

Reviewed-by: coleenp

Author: Harold Seigel

test/hotspot/jtreg/vmTestbase/nsk/sysdict/share/GenClassesBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -71,6 +71,7 @@ private static void build(String name, String prefix, Runnable generator) {
         moveJavaFiles(genSrcDir, prefix);
 
         JDKToolLauncher javac = JDKToolLauncher.create("javac")
+                                               .addToolArg("-J-Xmx1G")
                                                .addToolArg("-d")
                                                .addToolArg(classesDir.toString())
                                                .addToolArg("-cp")

test/hotspot/jtreg/vmTestbase/nsk/sysdict/share/SysDictTest.java

@@ -148,8 +148,8 @@ public void run() {
                     // set name into public variable just to be sure
                     // that class is loaded
                     tmp = clz.getName();
-                } catch (OutOfMemoryError | ClassNotFoundException e) {
-                    // just ignore
+                } catch (OutOfMemoryError | ClassNotFoundException | NoClassDefFoundError e) {
+                    // just ignore, note that CNFE and NCDFE can be caused by OOM exceptions.
                 } catch (StackOverflowError soe) {
                     // just ignore, chains could be too large
                     // StackOverflowError could be in some sparcs