8242338: Shape clip with AA not working in J2DDemo

src/java.desktop/macosx/native/libawt_lwawt/awt/shaders.metal

@@ -187,10 +187,17 @@ fragment half4 frag_txt_grad(GradShaderInOut in [[stage_in]],
 fragment half4 aa_frag_txt(
         TxtShaderInOut vert [[stage_in]],
         texture2d<float, access::sample> renderTexture [[texture(0)]],
+        texture2d<float, access::sample> stencilTexture [[texture(1)]],
         constant TxtFrameUniforms& uniforms [[buffer(1)]],
         sampler textureSampler [[sampler(0)]]
 ) {
     float4 pixelColor = renderTexture.sample(textureSampler, vert.texCoords);
+    if (!is_null_texture(stencilTexture)) {
+        float4 stencil = stencilTexture.sample(textureSampler, vert.texCoords);
+        if (stencil.r ==  0.0) {
+            discard_fragment();
+        }
+    }
     return half4(pixelColor.r, pixelColor.g, pixelColor.b, pixelColor.a);
 }
 

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/EncoderManager.m

@@ -411,12 +411,12 @@ - (void)setContext:(MTLContex * _Nonnull)mtlc {
       ca.storeAction = MTLStoreActionStore;
     }
 
-    if (_useStencil) {
-      // If you enable stencil testing or stencil writing, the
-      // MTLRenderPassDescriptor must include a stencil attachment.
-      rpd.stencilAttachment.texture = _mtlc.clip.stencilTextureRef;
-      rpd.stencilAttachment.loadAction = MTLLoadActionLoad;
-      rpd.stencilAttachment.storeAction = MTLStoreActionDontCare;
+    if (_useStencil && !isAA) {
+        // If you enable stencil testing or stencil writing, the
+        // MTLRenderPassDescriptor must include a stencil attachment.
+        rpd.stencilAttachment.loadAction = MTLLoadActionLoad;
+        rpd.stencilAttachment.storeAction = MTLStoreActionStore;
+        rpd.stencilAttachment.texture = _mtlc.clip.stencilTextureRef;
     }
 
     // J2dTraceLn1(J2D_TRACE_VERBOSE, "created render encoder to draw on
@@ -467,6 +467,7 @@ - (void) endEncoder {
                                  isDstOpaque:JNI_TRUE
                                interpolation:INTERPOLATION_NEAREST_NEIGHBOR
                                         isAA:JNI_TRUE];
+          [_encoder setFragmentTexture:_mtlc.clip.stencilAADataRef atIndex: 1];
 
           struct TxtVertex quadTxVerticesBuffer[] = {
               {{0, 0}, {0, 0}},

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLClip.h

@@ -22,6 +22,7 @@ enum Clip {
  * */
 
 @interface MTLClip : NSObject
+@property (readonly) id<MTLTexture> stencilAADataRef;
 @property (readonly) id<MTLTexture> stencilTextureRef;
 @property (readonly) jboolean stencilMaskGenerationInProgress;
 

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLClip.m

@@ -1,5 +1,4 @@
 #include "MTLClip.h"
-#import <iso646.h>
 
 #include "MTLContext.h"
 #include "common.h"
@@ -53,6 +52,7 @@ @implementation MTLClip {
 
     jboolean _stencilMaskGenerationInProgress;
     id<MTLTexture> _stencilTextureRef;
+    id<MTLTexture> _stencilAADataRef;
 }
 
 - (id)init {
@@ -99,6 +99,7 @@ - (void)copyFrom:(MTLClip *)other {
     _clipType = other->_clipType;
     _stencilMaskGenerationInProgress = other->_stencilMaskGenerationInProgress;
     _stencilTextureRef = other->_stencilTextureRef;
+    _stencilAADataRef = other->_stencilAADataRef;
     if (other->_clipType == RECT_CLIP) {
         _clipRect = other->_clipRect;
     }
@@ -107,12 +108,14 @@ - (void)copyFrom:(MTLClip *)other {
 - (void)reset {
     _clipType = NO_CLIP;
     _stencilTextureRef = nil;
+    _stencilAADataRef = nil;
     _stencilMaskGenerationInProgress = JNI_FALSE;
 }
 
 - (void)setClipRectX1:(jint)x1 Y1:(jint)y1 X2:(jint)x2 Y2:(jint)y2 {
     if (_clipType == SHAPE_CLIP) {
         _stencilTextureRef = nil;
+        _stencilAADataRef = nil;
     }
 
     if (x1 >= x2 || y1 >= y2) {
@@ -148,31 +151,44 @@ - (void)beginShapeClip:(BMTLSDOps *)dstOps context:(MTLContext *)mtlc {
 
         NSUInteger width = (NSUInteger)dstOps->width;
         NSUInteger height = (NSUInteger)dstOps->height;
-        id <MTLBuffer> buff = [mtlc.device newBufferWithLength:width * height options:MTLResourceStorageModeShared];
-        memset(buff.contents, 0, width * height);
-
+        NSUInteger size = width*height;
+        id <MTLBuffer> buff = [mtlc.device newBufferWithLength:size*4 options:MTLResourceStorageModePrivate];
         id<MTLCommandBuffer> commandBuf = [mtlc createBlitCommandBuffer];
         id<MTLBlitCommandEncoder> blitEncoder = [commandBuf blitCommandEncoder];
+        [blitEncoder fillBuffer:buff range:NSMakeRange(0, size*4) value:0];
 
+        MTLOrigin origin = MTLOriginMake(0, 0, 0);
+        MTLSize sourceSize = MTLSizeMake(width, height, 1);
         [blitEncoder copyFromBuffer:buff
                        sourceOffset:0
                   sourceBytesPerRow:width
-                sourceBytesPerImage:width * height
-                         sourceSize:MTLSizeMake(width, height, 1)
+                sourceBytesPerImage:size
+                         sourceSize:sourceSize
                           toTexture:dstOps->pStencilData
                    destinationSlice:0
                    destinationLevel:0
-                  destinationOrigin:MTLOriginMake(0, 0, 0)];
+                  destinationOrigin:origin];
+
+        [blitEncoder copyFromBuffer:buff
+                       sourceOffset:0
+                  sourceBytesPerRow:width*4
+                sourceBytesPerImage:size*4
+                         sourceSize:sourceSize
+                          toTexture:dstOps->pAAStencilData
+                   destinationSlice:0
+                   destinationLevel:0
+                  destinationOrigin:origin];
 
         [blitEncoder copyFromBuffer:buff
                        sourceOffset:0
                   sourceBytesPerRow:width
-                sourceBytesPerImage:width * height
-                         sourceSize:MTLSizeMake(width, height, 1)
+                sourceBytesPerImage:size
+                         sourceSize:sourceSize
                           toTexture:dstOps->pStencilTexture
                    destinationSlice:0
                    destinationLevel:0
-                  destinationOrigin:MTLOriginMake(0, 0, 0)];
+                  destinationOrigin:origin];
+
         [blitEncoder endEncoding];
 
         [commandBuf commit];
@@ -205,46 +221,76 @@ - (void)endShapeClip:(BMTLSDOps *)dstOps context:(MTLContext *)mtlc {
     // Now the stencil data is ready, this needs to be used while rendering further
     @autoreleasepool {
         if (dstOps->width > 0 && dstOps->height > 0) {
-          NSUInteger width = (NSUInteger)dstOps->width;
-          NSUInteger height = (NSUInteger)dstOps->height;
-
-          id<MTLBuffer> buff =
-              [mtlc.device newBufferWithLength:width * height
-                                       options:MTLResourceStorageModeShared];
-
-          id<MTLCommandBuffer> cb = [mtlc createBlitCommandBuffer];
-          id<MTLBlitCommandEncoder> blitEncoder = [cb blitCommandEncoder];
-          [blitEncoder copyFromTexture:dstOps->pStencilData
-                           sourceSlice:0
-                           sourceLevel:0
-                          sourceOrigin:MTLOriginMake(0, 0, 0)
-                            sourceSize:MTLSizeMake(width, height, 1)
-                              toBuffer:buff
-                     destinationOffset:0
-                destinationBytesPerRow:width
-              destinationBytesPerImage:width * height];
-
-          [blitEncoder copyFromBuffer:buff
-                         sourceOffset:0
-                    sourceBytesPerRow:width
-                  sourceBytesPerImage:width * height
-                           sourceSize:MTLSizeMake(width, height, 1)
-                            toTexture:dstOps->pStencilTexture
-                     destinationSlice:0
-                     destinationLevel:0
-                    destinationOrigin:MTLOriginMake(0, 0, 0)];
-
-          [blitEncoder endEncoding];
-
-          [cb commit];
-          [cb waitUntilCompleted];
-
-          [buff release];
+            NSUInteger width = (NSUInteger)dstOps->width;
+            NSUInteger height = (NSUInteger)dstOps->height;
+            NSUInteger size = width*height;
+            NSUInteger sizeX4 = size*4;
+
+            id<MTLBuffer> buff = 
+                [mtlc.device newBufferWithLength:size 
+                                         options:MTLResourceStorageModeShared];
+            id<MTLBuffer> aaBuff = 
+                [mtlc.device newBufferWithLength:sizeX4
+                                         options:MTLResourceStorageModeShared];
+
+            id<MTLCommandBuffer> cb = [mtlc createBlitCommandBuffer];
+            id<MTLBlitCommandEncoder> blitEncoder = [cb blitCommandEncoder];
+            MTLSize sourceSize = MTLSizeMake(width, height, 1);
+            MTLOrigin origin = MTLOriginMake(0, 0, 0);
+            [blitEncoder copyFromTexture:dstOps->pStencilData
+                             sourceSlice:0
+                             sourceLevel:0
+                            sourceOrigin:origin
+                              sourceSize:sourceSize
+                                toBuffer:buff
+                       destinationOffset:0
+                  destinationBytesPerRow:width
+                destinationBytesPerImage:size];
+
+            [blitEncoder copyFromBuffer:buff
+                           sourceOffset:0
+                      sourceBytesPerRow:width
+                    sourceBytesPerImage:size
+                             sourceSize:sourceSize
+                              toTexture:dstOps->pStencilTexture
+                       destinationSlice:0
+                       destinationLevel:0
+                      destinationOrigin:origin];
+
+            [blitEncoder endEncoding];
+            [cb commit];
+            [cb waitUntilCompleted];
+// TODO: Implement via compute shader
+            for (int i = 0; i < width*height; i++) {
+                unsigned char c =  ((unsigned char*)(buff.contents))[i];
+                ((jint*)(aaBuff.contents))[i] = c + (c << 8) + (c << 16) + (c << 24);
+            }
+
+            cb = [mtlc createBlitCommandBuffer];
+            blitEncoder = [cb blitCommandEncoder];
+
+            [blitEncoder copyFromBuffer:aaBuff
+                           sourceOffset:0
+                      sourceBytesPerRow:width*4
+                    sourceBytesPerImage:sizeX4
+                             sourceSize:sourceSize
+                              toTexture:dstOps->pAAStencilData
+                       destinationSlice:0
+                       destinationLevel:0
+                      destinationOrigin:origin];
+            [blitEncoder endEncoding];
+
+            [cb commit];
+            [cb waitUntilCompleted];
+
+            [buff release];
+            [aaBuff release];
         }
     }
 
     _stencilMaskGenerationInProgress = JNI_FALSE;
     _stencilTextureRef = dstOps->pStencilTexture;
+    _stencilAADataRef = dstOps->pAAStencilData;
     _clipType = SHAPE_CLIP;
 }
 

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLRenderer.m

@@ -283,7 +283,8 @@ void MTLRenderer_FillSpans(MTLContext *mtlc, BMTLSDOps * dstOps, jint spanCount,
 
     // Destination texture to which render commands are encoded
     id<MTLTexture> dest = dstOps->pTexture;
-    bool isDestOpaque = dstOps->isOpaque;
+    id<MTLTexture> destAA = nil;
+    BOOL isDestOpaque = dstOps->isOpaque;
     if (mtlc.clip.stencilMaskGenerationInProgress == JNI_TRUE) {
         dest = dstOps->pStencilData;
         isDestOpaque = NO;
@@ -299,7 +300,7 @@ void MTLRenderer_FillSpans(MTLContext *mtlc, BMTLSDOps * dstOps, jint spanCount,
     struct Vertex vertexList[TOTAL_VERTICES_IN_BLOCK]; // a total of 170 triangles ==> 85 spans
 
     int counter = 0;
-
+    jint *aaspans = spans;
     for (int i = 0; i < spanCount; i++) {
         jfloat x1 = *(spans++);
         jfloat y1 = *(spans++);

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLSurfaceData.m

@@ -77,6 +77,8 @@ static jboolean MTLSurfaceData_initTexture(BMTLSDOps *bmtlsdo, jboolean isOpaque
         stencilDataDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead;
         stencilDataDescriptor.storageMode = MTLStorageModePrivate;
         bmtlsdo->pStencilData = [ctx.device newTextureWithDescriptor:stencilDataDescriptor];
+        bmtlsdo->pAAStencilData = [ctx.device newTextureWithDescriptor:textureDescriptor];
+
 
         MTLTextureDescriptor *stencilTextureDescriptor =
             [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatStencil8 width:width height:height mipmapped:NO];

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLSurfaceDataBase.h

@@ -173,7 +173,8 @@ typedef struct {
     jint                         height;
     void*                        pTexture;
     void*                        pStencilData; // stencil data to be rendered to this buffer
-    void*                        pStencilTexture; // byte buffer stencil mask used in main rendering
+    void*                        pStencilTexture; // stencil texture byte buffer stencil mask used in main rendering
+    void*                        pAAStencilData; // stencil data for AA rendering
     void*                        textureLCD;
     jint                         textureWidth;
     jint                         textureHeight;