8274033: Some tier-4 CDS EpsilonGC tests throw OOM

Reviewed-by: dholmes, ccheung, minqi

Author: iklam

test/hotspot/jtreg/TEST.groups

@@ -399,6 +399,19 @@ hotspot_cds_verify_shared_spaces = \
   runtime/cds/appcds/jvmti/dumpingWithAgent/DumpingWithJavaAgent.java \
   runtime/cds/appcds/sharedStrings/SharedStringsBasic.java
 
+# No need to run every test with EpsilonGC. A small subset will provide enough
+# coverage. Also avoid some tests that may OOM.
+hotspot_cds_epsilongc = \
+  runtime/cds/appcds/HelloTest.java \
+  runtime/cds/appcds/MultiProcessSharing.java \
+  runtime/cds/appcds/StaticArchiveWithLambda.java \
+  runtime/cds/appcds/cacheObject \
+  runtime/cds/appcds/customLoader/ParallelTestSingleFP.java \
+  runtime/cds/appcds/dynamicArchive/HelloDynamic.java \
+  runtime/cds/appcds/javaldr \
+  runtime/cds/appcds/jigsaw \
+  runtime/cds/appcds/loaderConstraints
+
 # needs -nativepath:<output>/images/test/hotspot/jtreg/native/
 hotspot_metaspace = \
   gtest/MetaspaceGtests.java \