Skip to content

Commit fc92adc

Browse files
Andrew LeonardRealCLanger
Andrew Leonard
authored and
RealCLanger
committedJan 14, 2022
8276764: Enable deterministic file content ordering for Jar and Jmod
Backport-of: 24e586a04368a76cd9f37aa783b974b9e0351d58
·
jdk-17.0.16+0jdk-17.0.3-ga
1 parent 0130fdc commit fc92adc

File tree

4 files changed

+263
-15
lines changed

4 files changed

+263
-15
lines changed
 

‎src/jdk.jartool/share/classes/sun/tools/jar/Main.java

+21-9
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1996, 2018, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 1996, 2021, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -127,7 +127,10 @@ public int hashCode() {
127127
Map<Integer,Set<String>> pathsMap = new HashMap<>();
128128

129129
// There's also a files array per version
130-
Map<Integer,String[]> filesMap = new HashMap<>();
130+
// base version is the first entry and then follow with the version given
131+
// from the --release option in the command-line order.
132+
// The value of each entry is the files given in the command-line order.
133+
Map<Integer,String[]> filesMap = new LinkedHashMap<>();
131134

132135
// Do we think this is a multi-release jar? Set to true
133136
// if --release option found followed by at least file
@@ -772,15 +775,17 @@ private void expand() throws IOException {
772775
private void expand(File dir, String[] files, Set<String> cpaths, int version)
773776
throws IOException
774777
{
775-
if (files == null)
778+
if (files == null) {
776779
return;
780+
}
777781

778782
for (int i = 0; i < files.length; i++) {
779783
File f;
780-
if (dir == null)
784+
if (dir == null) {
781785
f = new File(files[i]);
782-
else
786+
} else {
783787
f = new File(dir, files[i]);
788+
}
784789

785790
boolean isDir = f.isDirectory();
786791
String name = toEntryName(f.getPath(), cpaths, isDir);
@@ -802,18 +807,20 @@ private void expand(File dir, String[] files, Set<String> cpaths, int version)
802807
Entry e = new Entry(f, name, false);
803808
if (isModuleInfoEntry(name)) {
804809
moduleInfos.putIfAbsent(name, Files.readAllBytes(f.toPath()));
805-
if (uflag)
810+
if (uflag) {
806811
entryMap.put(name, e);
812+
}
807813
} else if (entries.add(e)) {
808-
if (uflag)
814+
if (uflag) {
809815
entryMap.put(name, e);
816+
}
810817
}
811818
} else if (isDir) {
812819
Entry e = new Entry(f, name, true);
813820
if (entries.add(e)) {
814821
// utilize entryMap for the duplicate dir check even in
815822
// case of cflag == true.
816-
// dir name confilict/duplicate could happen with -C option.
823+
// dir name conflict/duplicate could happen with -C option.
817824
// just remove the last "e" from the "entries" (zos will fail
818825
// with "duplicated" entries), but continue expanding the
819826
// sub tree
@@ -822,7 +829,12 @@ private void expand(File dir, String[] files, Set<String> cpaths, int version)
822829
} else {
823830
entryMap.put(name, e);
824831
}
825-
expand(f, f.list(), cpaths, version);
832+
String[] dirFiles = f.list();
833+
// Ensure files list is sorted for reproducible jar content
834+
if (dirFiles != null) {
835+
Arrays.sort(dirFiles);
836+
}
837+
expand(f, dirFiles, cpaths, version);
826838
}
827839
} else {
828840
error(formatMsg("error.nosuch.fileordir", String.valueOf(f)));

‎src/jdk.jlink/share/classes/jdk/tools/jmod/JmodTask.java

+15-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -767,6 +767,10 @@ void processSection(JmodOutputStream out, Section section, List<Path> paths)
767767
void processSection(JmodOutputStream out, Section section, Path path)
768768
throws IOException
769769
{
770+
// Keep a sorted set of files to be processed, so that the jmod
771+
// content is reproducible as Files.walkFileTree order is not defined
772+
SortedMap<String, Path> filesToProcess = new TreeMap<String, Path>();
773+
770774
Files.walkFileTree(path, Set.of(FileVisitOption.FOLLOW_LINKS),
771775
Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
772776
@Override
@@ -782,14 +786,21 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
782786
if (out.contains(section, name)) {
783787
warning("warn.ignore.duplicate.entry", name, section);
784788
} else {
785-
try (InputStream in = Files.newInputStream(file)) {
786-
out.writeEntry(in, section, name);
787-
}
789+
filesToProcess.put(name, file);
788790
}
789791
}
790792
return FileVisitResult.CONTINUE;
791793
}
792794
});
795+
796+
// Process files in sorted order for deterministic jmod content
797+
for (Map.Entry<String, Path> entry : filesToProcess.entrySet()) {
798+
String name = entry.getKey();
799+
Path file = entry.getValue();
800+
try (InputStream in = Files.newInputStream(file)) {
801+
out.writeEntry(in, section, name);
802+
}
803+
}
793804
}
794805

795806
boolean matches(Path path, List<PathMatcher> matchers) {

‎test/jdk/tools/jar/ContentOrder.java

+214
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
/*
2+
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* This code is free software; you can redistribute it and/or modify it
6+
* under the terms of the GNU General Public License version 2 only, as
7+
* published by the Free Software Foundation.
8+
*
9+
* This code is distributed in the hope that it will be useful, but WITHOUT
10+
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11+
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12+
* version 2 for more details (a copy is included in the LICENSE file that
13+
* accompanied this code).
14+
*
15+
* You should have received a copy of the GNU General Public License version
16+
* 2 along with this work; if not, write to the Free Software Foundation,
17+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18+
*
19+
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20+
* or visit www.oracle.com if you need additional information or have any
21+
* questions.
22+
*/
23+
24+
/*
25+
* @test
26+
* @bug 8276764
27+
* @summary test that the jar content ordering is sorted
28+
* @library /test/lib
29+
* @modules jdk.jartool
30+
* @build jdk.test.lib.Platform
31+
* jdk.test.lib.util.FileUtils
32+
* @run testng ContentOrder
33+
*/
34+
35+
import org.testng.Assert;
36+
import org.testng.annotations.AfterMethod;
37+
import org.testng.annotations.BeforeMethod;
38+
import org.testng.annotations.Test;
39+
40+
import java.io.ByteArrayOutputStream;
41+
import java.io.IOException;
42+
import java.io.PrintStream;
43+
import java.io.UncheckedIOException;
44+
import java.io.File;
45+
import java.nio.file.Files;
46+
import java.nio.file.Path;
47+
import java.nio.file.Paths;
48+
import java.util.Arrays;
49+
import java.util.spi.ToolProvider;
50+
import java.util.stream.Stream;
51+
import java.util.zip.ZipException;
52+
53+
import jdk.test.lib.util.FileUtils;
54+
55+
public class ContentOrder {
56+
private static final ToolProvider JAR_TOOL = ToolProvider.findFirst("jar")
57+
.orElseThrow(() ->
58+
new RuntimeException("jar tool not found")
59+
);
60+
61+
private final String nl = System.lineSeparator();
62+
private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
63+
private final PrintStream out = new PrintStream(baos);
64+
private Runnable onCompletion;
65+
66+
@BeforeMethod
67+
public void reset() {
68+
onCompletion = null;
69+
}
70+
71+
@AfterMethod
72+
public void run() {
73+
if (onCompletion != null) {
74+
onCompletion.run();
75+
}
76+
}
77+
78+
// Test that the jar content ordering when processing a single directory is sorted
79+
@Test
80+
public void testSingleDir() throws IOException {
81+
mkdir("testjar/Ctest1", "testjar/Btest2/subdir1", "testjar/Atest3");
82+
touch("testjar/Ctest1/testfile1", "testjar/Ctest1/testfile2", "testjar/Ctest1/testfile3");
83+
touch("testjar/Btest2/subdir1/testfileC", "testjar/Btest2/subdir1/testfileB", "testjar/Btest2/subdir1/testfileA");
84+
touch("testjar/Atest3/fileZ", "testjar/Atest3/fileY", "testjar/Atest3/fileX");
85+
86+
onCompletion = () -> rm("test.jar", "testjar");
87+
88+
jar("cf test.jar testjar");
89+
jar("tf test.jar");
90+
System.out.println(new String(baos.toByteArray()));
91+
String output = "META-INF/" + nl +
92+
"META-INF/MANIFEST.MF" + nl +
93+
"testjar/" + nl +
94+
"testjar/Atest3/" + nl +
95+
"testjar/Atest3/fileX" + nl +
96+
"testjar/Atest3/fileY" + nl +
97+
"testjar/Atest3/fileZ" + nl +
98+
"testjar/Btest2/" + nl +
99+
"testjar/Btest2/subdir1/" + nl +
100+
"testjar/Btest2/subdir1/testfileA" + nl +
101+
"testjar/Btest2/subdir1/testfileB" + nl +
102+
"testjar/Btest2/subdir1/testfileC" + nl +
103+
"testjar/Ctest1/" + nl +
104+
"testjar/Ctest1/testfile1" + nl +
105+
"testjar/Ctest1/testfile2" + nl +
106+
"testjar/Ctest1/testfile3" + nl;
107+
Assert.assertEquals(baos.toByteArray(), output.getBytes());
108+
}
109+
110+
// Test that when specifying multiple directories or releases that the sort
111+
// ordering is done on each directory and release, reserving the order of
112+
// the directories/releases specified on the command line
113+
@Test
114+
public void testMultiDirWithReleases() throws IOException {
115+
mkdir("testjar/foo/classes",
116+
"testjar/foo11/classes/Zclasses",
117+
"testjar/foo11/classes/Yclasses",
118+
"testjar/foo17/classes/Bclasses",
119+
"testjar/foo17/classes/Aclasses");
120+
touch("testjar/foo/classes/testfile1", "testjar/foo/classes/testfile2");
121+
touch("testjar/foo11/classes/Zclasses/testfile1", "testjar/foo11/classes/Zclasses/testfile2");
122+
touch("testjar/foo11/classes/Yclasses/testfileA", "testjar/foo11/classes/Yclasses/testfileB");
123+
touch("testjar/foo17/classes/Bclasses/testfile1", "testjar/foo17/classes/Bclasses/testfile2");
124+
touch("testjar/foo17/classes/Aclasses/testfileA", "testjar/foo17/classes/Aclasses/testfileB");
125+
126+
onCompletion = () -> rm("test.jar", "testjar");
127+
128+
jar("cf test.jar -C testjar/foo classes " +
129+
"--release 17 -C testjar/foo17 classes/Bclasses -C testjar/foo17 classes/Aclasses " +
130+
"--release 11 -C testjar/foo11 classes/Zclasses -C testjar/foo11 classes/Yclasses");
131+
jar("tf test.jar");
132+
System.out.println(new String(baos.toByteArray()));
133+
String output = "META-INF/" + nl +
134+
"META-INF/MANIFEST.MF" + nl +
135+
"classes/" + nl +
136+
"classes/testfile1" + nl +
137+
"classes/testfile2" + nl +
138+
"META-INF/versions/17/classes/Bclasses/" + nl +
139+
"META-INF/versions/17/classes/Bclasses/testfile1" + nl +
140+
"META-INF/versions/17/classes/Bclasses/testfile2" + nl +
141+
"META-INF/versions/17/classes/Aclasses/" + nl +
142+
"META-INF/versions/17/classes/Aclasses/testfileA" + nl +
143+
"META-INF/versions/17/classes/Aclasses/testfileB" + nl +
144+
"META-INF/versions/11/classes/Zclasses/" + nl +
145+
"META-INF/versions/11/classes/Zclasses/testfile1" + nl +
146+
"META-INF/versions/11/classes/Zclasses/testfile2" + nl +
147+
"META-INF/versions/11/classes/Yclasses/" + nl +
148+
"META-INF/versions/11/classes/Yclasses/testfileA" + nl +
149+
"META-INF/versions/11/classes/Yclasses/testfileB" + nl;
150+
Assert.assertEquals(baos.toByteArray(), output.getBytes());
151+
}
152+
153+
private Stream<Path> mkpath(String... args) {
154+
return Arrays.stream(args).map(d -> Paths.get(".", d.split("/")));
155+
}
156+
157+
private void mkdir(String... dirs) {
158+
System.out.println("mkdir -p " + Arrays.toString(dirs));
159+
Arrays.stream(dirs).forEach(p -> {
160+
try {
161+
Files.createDirectories((new File(p)).toPath());
162+
} catch (IOException x) {
163+
throw new UncheckedIOException(x);
164+
}
165+
});
166+
}
167+
168+
private void touch(String... files) {
169+
System.out.println("touch " + Arrays.toString(files));
170+
Arrays.stream(files).forEach(p -> {
171+
try {
172+
Files.createFile((new File(p)).toPath());
173+
} catch (IOException x) {
174+
throw new UncheckedIOException(x);
175+
}
176+
});
177+
}
178+
179+
private void rm(String... files) {
180+
System.out.println("rm -rf " + Arrays.toString(files));
181+
Arrays.stream(files).forEach(p -> {
182+
try {
183+
Path path = (new File(p)).toPath();
184+
if (Files.isDirectory(path)) {
185+
FileUtils.deleteFileTreeWithRetry(path);
186+
} else {
187+
FileUtils.deleteFileIfExistsWithRetry(path);
188+
}
189+
} catch (IOException x) {
190+
throw new UncheckedIOException(x);
191+
}
192+
});
193+
}
194+
195+
private void jar(String cmdline) throws IOException {
196+
System.out.println("jar " + cmdline);
197+
baos.reset();
198+
199+
// the run method catches IOExceptions, we need to expose them
200+
ByteArrayOutputStream baes = new ByteArrayOutputStream();
201+
PrintStream err = new PrintStream(baes);
202+
PrintStream saveErr = System.err;
203+
System.setErr(err);
204+
int rc = JAR_TOOL.run(out, err, cmdline.split(" +"));
205+
System.setErr(saveErr);
206+
if (rc != 0) {
207+
String s = baes.toString();
208+
if (s.startsWith("java.util.zip.ZipException: duplicate entry: ")) {
209+
throw new ZipException(s);
210+
}
211+
throw new IOException(s);
212+
}
213+
}
214+
}

‎test/jdk/tools/jmod/JmodTest.java

+13-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,7 @@
2323

2424
/*
2525
* @test
26-
* @bug 8142968 8166568 8166286 8170618 8168149 8240910
26+
* @bug 8142968 8166568 8166286 8170618 8168149 8240910 8276764
2727
* @summary Basic test for jmod
2828
* @library /test/lib
2929
* @modules jdk.compiler
@@ -197,6 +197,17 @@ public void testList() throws IOException {
197197
assertContains(r.output, CLASSES_PREFIX + "jdk/test/foo/Foo.class");
198198
assertContains(r.output, CLASSES_PREFIX + "jdk/test/foo/internal/Message.class");
199199
assertContains(r.output, CLASSES_PREFIX + "jdk/test/foo/resources/foo.properties");
200+
201+
// JDK-8276764: Ensure the order is sorted for reproducible jmod content
202+
// module-info, followed by <sorted classes>
203+
int mod_info_i = r.output.indexOf(CLASSES_PREFIX + "module-info.class");
204+
int foo_cls_i = r.output.indexOf(CLASSES_PREFIX + "jdk/test/foo/Foo.class");
205+
int msg_i = r.output.indexOf(CLASSES_PREFIX + "jdk/test/foo/internal/Message.class");
206+
int res_i = r.output.indexOf(CLASSES_PREFIX + "jdk/test/foo/resources/foo.properties");
207+
System.out.println("jmod classes sort order check:\n"+r.output);
208+
assertTrue(mod_info_i < foo_cls_i);
209+
assertTrue(foo_cls_i < msg_i);
210+
assertTrue(msg_i < res_i);
200211
});
201212
}
202213

1 commit comments

Comments
 (1)

openjdk-notifier[bot] commented on Jan 14, 2022

@openjdk-notifier[bot]
openjdk-notifier[bot]
on Jan 14, 2022

Review

Issues

Please sign in to comment.