JDK-8238535: Shape Clip does not work with AA rendering

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/EncoderManager.m

@@ -16,7 +16,6 @@ - (id)init;
 - (void)dealloc;
 
 - (void)reset:(id<MTLTexture>)destination
-                aaDest:(id<MTLTexture>)aaDestination
            isDstOpaque:(jboolean)isDstOpaque
     isDstPremultiplied:(jboolean)isDstPremultiplied
                   isAA:(jboolean)isAA;
@@ -39,7 +38,6 @@ @implementation EncoderStates {
 
     // Persistent encoder properties
     id<MTLTexture> _destination;
-    id<MTLTexture> _aaDestination;
     SurfaceRasterFlags _dstFlags;
 
     //
@@ -69,7 +67,6 @@ - (id)init {
     self = [super init];
     if (self) {
         _destination = nil;
-        _aaDestination = nil;
         _composite = [[MTLComposite alloc] init];
         _paint = [[MTLPaint alloc] init];
         _transform = [[MTLTransform alloc] init];
@@ -91,13 +88,10 @@ - (void)setContext:(MTLContext * _Nonnull)mtlc {
 }
 
 - (void)reset:(id<MTLTexture>)destination
-                aaDest:(id<MTLTexture>)aaDestination
            isDstOpaque:(jboolean)isDstOpaque
     isDstPremultiplied:(jboolean)isDstPremultiplied
-                  isAA:(jboolean)isAA
-{
+                  isAA:(jboolean)isAA {
     _destination = destination;
-    _aaDestination = aaDestination;
     _dstFlags.isOpaque = isDstOpaque;
     _dstFlags.isPremultiplied = isDstPremultiplied;
     _isAA = isAA;
@@ -204,17 +198,10 @@ - (void) updateClip:(id<MTLRenderCommandEncoder>)encoder clip:(MTLClip *)clip fo
         return;
 
     [_clip copyFrom:clip];
-    if (_aaDestination != nil) {
-      [_clip setScissorOrStencil:encoder
-                       destWidth:_aaDestination.width
-                      destHeight:_aaDestination.height
-                          device:_device];
-    } else {
-      [_clip setScissorOrStencil:encoder
-                       destWidth:_destination.width
-                      destHeight:_destination.height
-                          device:_device];
-    }
+    [_clip setScissorOrStencil:encoder
+                     destWidth:_destination.width
+                    destHeight:_destination.height
+                        device:_device];
 }
 
 - (void)updateTransform:(id <MTLRenderCommandEncoder>)encoder
@@ -226,16 +213,9 @@ - (void)updateTransform:(id <MTLRenderCommandEncoder>)encoder
         return;
 
     [_transform copyFrom:transform];
-    if (_aaDestination != nil) {
-      [_transform setVertexMatrix:encoder
-                        destWidth:_aaDestination.width
-                       destHeight:_aaDestination.height];
-    } else {
-      [_transform setVertexMatrix:encoder
+    [_transform setVertexMatrix:encoder
                         destWidth:_destination.width
                        destHeight:_destination.height];
-
-    }
 }
 
 @end
@@ -417,7 +397,6 @@ - (void)setContext:(MTLContex * _Nonnull)mtlc {
     [rpd release];
 
     [_encoderStates reset:dest
-               aaDest:_aaDestination
                isDstOpaque:isOpaque
         isDstPremultiplied:YES
                       isAA:isAA];

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLRenderQueue.h

@@ -42,6 +42,19 @@
 #define NEXT_LONG(buf)      NEXT_VAL(buf, jlong)
 #define NEXT_DOUBLE(buf)    NEXT_VAL(buf, jdouble)
 
+// Operations for CheckPreviousOp
+enum {
+  MTL_OP_INIT,
+  MTL_OP_AA,
+  MTL_OP_OTHER
+};
+
+/*
+ * These macros now simply delegate to the CheckPreviousOp() method.
+ */
+#define CHECK_PREVIOUS_OP(op) MTLRenderQueue_CheckPreviousOp(op)
+#define RESET_PREVIOUS_OP() CHECK_PREVIOUS_OP(MTL_OP_INIT)
+
 /*
  * Increments a pointer (buf) by the given number of bytes.
  */
@@ -74,4 +87,6 @@
 MTLContext *MTLRenderQueue_GetCurrentContext();
 BMTLSDOps *MTLRenderQueue_GetCurrentDestination();
 
+extern jint mtlPreviousOp;
+
 #endif /* MTLRenderQueue_h_Included */