From b2b03aade80f79948c380160480a9b4ff6fab404 Mon Sep 17 00:00:00 2001 From: knaccc Date: Sun, 30 Jun 2019 04:56:27 +0100 Subject: [PATCH] First attempt at reproducible builds. Seems to work on Linux/Mac, but Windows distributables are not yet producing consistent hashes due to jlink not currently being fully deterministic. Hopefully this will change in future JDK releases. --- bin/build-all-and-zip.sh | 10 + bin/build-all.sh | 3 + bin/build-launcher.sh | 17 +- bin/build-normalize-zip.sh | 18 ++ bin/clean-build-env.sh | 9 + bin/convert-jars-to-modules.sh | 3 + bin/import-packages.sh | 5 + bin/util.sh | 17 ++ bin/zip-all.sh | 43 ++-- .../org.getmonero.util.normalizeZip.iml | 20 ++ .../src/module-info.java | 3 + .../util/normalizeZip/CompoundStripper.java | 61 ++++++ .../util/normalizeZip/ManifestStripper.java | 67 ++++++ .../util/normalizeZip/NormalizeZip.java | 199 ++++++++++++++++++ .../SortManifestFileStripper.java | 114 ++++++++++ .../getmonero/util/normalizeZip/Stripper.java | 32 +++ .../util/normalizeZip/TextFileStripper.java | 64 ++++++ 17 files changed, 663 insertions(+), 22 deletions(-) create mode 100755 bin/build-all-and-zip.sh create mode 100755 bin/build-normalize-zip.sh create mode 100755 bin/clean-build-env.sh create mode 100644 bin/util.sh create mode 100644 org.getmonero.util.normalizeZip/org.getmonero.util.normalizeZip.iml create mode 100644 org.getmonero.util.normalizeZip/src/module-info.java create mode 100644 org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/CompoundStripper.java create mode 100644 org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/ManifestStripper.java create mode 100644 org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/NormalizeZip.java create mode 100644 org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/SortManifestFileStripper.java create mode 100644 org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/Stripper.java create mode 100644 org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/TextFileStripper.java diff --git a/bin/build-all-and-zip.sh b/bin/build-all-and-zip.sh new file mode 100755 index 0000000..a217482 --- /dev/null +++ b/bin/build-all-and-zip.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ $(uname -s) = Darwin ]; then + basedir=$(dirname $(cd "$(dirname "$0")"; pwd -P)) +else + basedir=$(dirname $(dirname $(readlink -fm $0))) +fi + +"$basedir"/bin/build-all.sh +"$basedir"/bin/zip-all.sh \ No newline at end of file diff --git a/bin/build-all.sh b/bin/build-all.sh index 10b7252..a7f98d2 100755 --- a/bin/build-all.sh +++ b/bin/build-all.sh @@ -11,6 +11,9 @@ rm -fr "$basedir/target" "$basedir/dist" # retrieve the I2P Java sources, OpenJDK and the Ant build tool "$basedir"/bin/import-packages.sh +# build tool that normalizes zip/jar files for reproducible builds +"$basedir"/bin/build-normalize-zip.sh + # build the i2p project retrieved from the I2P repository "$basedir"/bin/build-original-i2p.sh diff --git a/bin/build-launcher.sh b/bin/build-launcher.sh index 2bb07f4..4d5abbc 100755 --- a/bin/build-launcher.sh +++ b/bin/build-launcher.sh @@ -7,7 +7,7 @@ else fi source "$basedir/bin/java-config.sh" - +source "$basedir/bin/util.sh" echo "*** Compiling CLI" "$JAVA_HOME"/bin/javac --module-path target/modules/combined.jar -d target/classes/org.getmonero.i2p.zero $(find org.getmonero.i2p.zero/src -name '*.java') @@ -15,6 +15,7 @@ cp org.getmonero.i2p.zero/src/org/getmonero/i2p/zero/VERSION target/classes/org. echo "*** Packaging CLI as a modular jar" "$JAVA_HOME"/bin/jar --create --file target/org.getmonero.i2p.zero.jar --main-class org.getmonero.i2p.zero.Main -C target/classes/org.getmonero.i2p.zero . +normalizeZip target/org.getmonero.i2p.zero.jar echo "*** Compiling GUI" "$JAVA_HOME"/bin/javac --module-path target/org.getmonero.i2p.zero.jar:target/modules/combined.jar:import/javafx-sdks/linux/javafx-sdk-$JAVAFX_VERSION/lib -d target/classes/org.getmonero.i2p.zero.gui $(find org.getmonero.i2p.zero.gui/src -name '*.java') @@ -23,7 +24,7 @@ cp -r org.getmonero.i2p.zero.gui/src/org/getmonero/i2p/zero/gui/*.{css,png,fxml, echo "*** Packaging GUI as a modular jar" "$JAVA_HOME"/bin/jar --create --file target/org.getmonero.i2p.zero.gui.jar --main-class org.getmonero.i2p.zero.gui.Gui -C target/classes/org.getmonero.i2p.zero.gui . - +normalizeZip target/org.getmonero.i2p.zero.gui.jar rm -fr "$basedir/dist" for i in linux mac win linux-gui mac-gui win-gui; do mkdir -p "$basedir/dist/$i"; done @@ -41,8 +42,8 @@ for i in linux mac win; do JAVA_HOME_VARIANT=${JAVA_HOME_WIN} ;; esac echo "Using JAVA_HOME_VARIANT: $JAVA_HOME_VARIANT" - "$JAVA_HOME"/bin/jlink --module-path "${JAVA_HOME_VARIANT}/jmods":target/modules:target/org.getmonero.i2p.zero.jar --add-modules combined,org.getmonero.i2p.zero --output dist/$i/router --compress 2 --no-header-files --no-man-pages - "$JAVA_HOME"/bin/jlink --module-path "${JAVA_HOME_VARIANT}/jmods":import/javafx-jmods/$i/javafx-jmods-${JAVAFX_VERSION}:target/modules:target/org.getmonero.i2p.zero.jar:target/org.getmonero.i2p.zero.gui.jar --add-modules combined,org.getmonero.i2p.zero,org.getmonero.i2p.zero.gui,javafx.controls,javafx.fxml,java.desktop --output dist/$i-gui/router --compress 2 --no-header-files --no-man-pages + "$JAVA_HOME"/bin/jlink --module-path "${JAVA_HOME_VARIANT}/jmods":target/modules:target/org.getmonero.i2p.zero.jar --add-modules combined,org.getmonero.i2p.zero --output dist/$i/router --compress 2 --no-header-files --no-man-pages --order-resources=**/module-info.class,/java.base/java/lang/**,**javafx** + "$JAVA_HOME"/bin/jlink --module-path "${JAVA_HOME_VARIANT}/jmods":import/javafx-jmods/$i/javafx-jmods-${JAVAFX_VERSION}:target/modules:target/org.getmonero.i2p.zero.jar:target/org.getmonero.i2p.zero.gui.jar --add-modules combined,org.getmonero.i2p.zero,org.getmonero.i2p.zero.gui,javafx.controls,javafx.fxml,java.desktop --output dist/$i-gui/router --compress 2 --no-header-files --no-man-pages --order-resources=**/module-info.class,/java.base/java/lang/**,**javafx** done for i in linux mac linux-gui mac-gui; do @@ -57,7 +58,6 @@ for i in linux-gui mac-gui; do cp "$basedir/resources/launch-gui.sh" "$basedir/dist/$i/router/bin/" done - for i in linux mac win linux-gui mac-gui win-gui; do cp -r "$basedir/import/i2p.base" "$basedir/dist/$i/router/"; done # remove unnecessary native libs from jbigi.jar @@ -67,11 +67,14 @@ for i in linux mac win; do if [ "$j" = "mac" ]; then j="osx"; fi if [ "$j" = "win" ]; then j="windows"; fi zip -d "$basedir/dist/$i/router/i2p.base/jbigi.jar" *-${j}-* - zip -d "$basedir/dist/$i-gui/router/i2p.base/jbigi.jar" *-${j}-* + normalizeZip "$basedir/dist/$i/router/i2p.base/jbigi.jar" + cp "$basedir/dist/$i/router/i2p.base/jbigi.jar" "$basedir/dist/$i-gui/router/i2p.base/jbigi.jar" fi done done + + # build mac gui app structure mv "$basedir/dist/mac-gui/router" "$basedir/dist/mac-gui/router-tmp" mkdir -p "$basedir/dist/mac-gui/router/i2p-zero.app/Contents/MacOS/" @@ -121,6 +124,8 @@ du -sk dist/* | awk '{printf "%.1f MB %s\n",$1/1024,$2}' echo "*** Done ***" +echo "To build the distribution archives and show reproducible build SHA-256 hashes, type: bin/zip-all.sh" +echo "" os_name=`uname -s` if [ $os_name = Darwin ]; then os_name=mac diff --git a/bin/build-normalize-zip.sh b/bin/build-normalize-zip.sh new file mode 100755 index 0000000..c017ae7 --- /dev/null +++ b/bin/build-normalize-zip.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ $(uname -s) = Darwin ]; then + basedir=$(dirname $(cd "$(dirname "$0")"; pwd -P)) +else + basedir=$(dirname $(dirname $(readlink -fm $0))) +fi + +source "$basedir/bin/java-config.sh" +source "$basedir/bin/util.sh" + +echo "*** Compiling Zip normalizer utility" +"$JAVA_HOME"/bin/javac --module-path import/commons-compress-1.18/commons-compress-1.18.jar -d target/classes/org.getmonero.util.normalizeZip $(find org.getmonero.util.normalizeZip/src -name '*.java') + +echo "*** Packaging Zip normalizer as a modular jar" +"$JAVA_HOME"/bin/jar --create --file target/org.getmonero.util.normalizeZip.jar --main-class org.getmonero.util.normalizeZip.NormalizeZip -C target/classes/org.getmonero.util.normalizeZip . + + diff --git a/bin/clean-build-env.sh b/bin/clean-build-env.sh new file mode 100755 index 0000000..92aa25c --- /dev/null +++ b/bin/clean-build-env.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ $(uname -s) = Darwin ]; then + basedir=$(dirname $(cd "$(dirname "$0")"; pwd -P)) +else + basedir=$(dirname $(dirname $(readlink -fm $0))) +fi + +rm -fr "$basedir/dist-zip" "$basedir/dist-zip-staging" "$basedir/dist" "$basedir/target" "$basedir/import" "$basedir/out" \ No newline at end of file diff --git a/bin/convert-jars-to-modules.sh b/bin/convert-jars-to-modules.sh index c82eb6e..ca264b5 100755 --- a/bin/convert-jars-to-modules.sh +++ b/bin/convert-jars-to-modules.sh @@ -7,6 +7,7 @@ else fi source "$basedir/bin/java-config.sh" +source "$basedir/bin/util.sh" cp "$basedir"/import/jetty-lib/*.jar "$basedir/import/lib/" @@ -49,4 +50,6 @@ echo "*** Creating new combined modular jar" cp $combinedJarPath "$basedir/target/modules/" "$JAVA_HOME"/bin/jar uf "$basedir/target/modules/combined.jar" -C "$basedir/target/module-info/combined" module-info.class +normalizeZip target/modules/combined.jar + diff --git a/bin/import-packages.sh b/bin/import-packages.sh index f540153..657aca7 100755 --- a/bin/import-packages.sh +++ b/bin/import-packages.sh @@ -37,6 +37,11 @@ if [ ! -d "$basedir/import/apache-ant-1.10.5" ]; then tar zxvf apache-ant-1.10.5-bin.tar.gz fi +if [ ! -d "$basedir/import/commons-compress-1.18" ]; then + wget https://www.mirrorservice.org/sites/ftp.apache.org//commons/compress/binaries/commons-compress-1.18-bin.tar.gz + tar zxvf commons-compress-1.18-bin.tar.gz +fi + if [ ! -d "$basedir/import/jetty-lib" ]; then mkdir -p jetty-lib wget --directory-prefix=jetty-lib http://central.maven.org/maven2/org/eclipse/jetty/jetty-server/9.4.14.v20181114/jetty-server-9.4.14.v20181114.jar diff --git a/bin/util.sh b/bin/util.sh new file mode 100644 index 0000000..166d8dd --- /dev/null +++ b/bin/util.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# get the SHA-256 hash of the specified file +getHash () { + if [ $(uname -s) = Darwin ]; then + h=`shasum -a 256 $1 | awk '{print $1}'` + else + h=`sha256sum $1 | awk '{print $1}'` + fi + echo $h +} + +# normalizes the specified jar or zip for reproducible build. Enforces consistent zip file order and sets all timestamps to the last modified date of the VERSION file +normalizeZip () { + java --module-path "$basedir/import/commons-compress-1.18/commons-compress-1.18.jar":"$basedir/target/org.getmonero.util.normalizeZip.jar" \ + -m org.getmonero.util.normalizeZip "$basedir/org.getmonero.i2p.zero/src/org/getmonero/i2p/zero/VERSION" "$1" +} \ No newline at end of file diff --git a/bin/zip-all.sh b/bin/zip-all.sh index 7336356..d6e7049 100755 --- a/bin/zip-all.sh +++ b/bin/zip-all.sh @@ -6,18 +6,24 @@ else basedir=$(dirname $(dirname $(readlink -fm $0))) fi +source "$basedir/bin/util.sh" + +VERSION=$(head -n 1 "$basedir/org.getmonero.i2p.zero/src/org/getmonero/i2p/zero/VERSION") + rm -fr "$basedir/dist-zip-staging" mkdir -p "$basedir/dist-zip-staging" rm -fr "$basedir/dist-zip" mkdir -p "$basedir/dist-zip" -VERSION=$(head -n 1 "$basedir/org.getmonero.i2p.zero/src/org/getmonero/i2p/zero/VERSION") - cd "$basedir/dist" for i in linux linux-gui mac mac-gui win win-gui; do cp -r ${i} "$basedir"/dist-zip-staging/i2p-zero-${i}.v${VERSION}; done +versionDate=`date -r "$basedir"/org.getmonero.i2p.zero/src/org/getmonero/i2p/zero/VERSION +"%Y%m%d%H%M.%S"` + +find "$basedir"/dist-zip-staging -exec touch -t $versionDate {} \; + cd "$basedir/dist-zip-staging" for i in win win-gui; do zip -r9 "$basedir"/dist-zip/i2p-zero-${i}.v${VERSION}.zip i2p-zero-${i}.v${VERSION}; done @@ -31,21 +37,26 @@ fi cd .. -print3ColsJustified () { - printf "%-.14s %s" "$1 " "| " - printf "%-.26s %s" "$2 " "| " - printf "%s\n" "$3" +print4ColsJustified () { + printf "%-.12s %s" "$1 " "| " + printf "%-.23s %s" "$2 " "| " + printf "%-.21s %s" "$3 " "| " + printf "%s\n" "$4" } -getDirSizeMB () { - du -sk $1 | awk '{printf "%.1f",$1/1024,$2}' +getFileSizeMB () { + s=`du -sk $1 | awk '{printf "%.1f",$1/1024,$2}'` + echo $s } -print3ColsJustified "OS" "Uncompressed size (MB)" "Compressed size (MB)" -print3ColsJustified "------------------------" "------------------------" "------------------------" -print3ColsJustified "Mac" "`getDirSizeMB $basedir/dist/mac`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-mac.v${VERSION}.tar.bz2`" -print3ColsJustified "Windows" "`getDirSizeMB $basedir/dist/win`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-win.v${VERSION}.zip`" -print3ColsJustified "Linux" "`getDirSizeMB $basedir/dist/linux`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-linux.v${VERSION}.tar.bz2`" -print3ColsJustified "Mac GUI" "`getDirSizeMB $basedir/dist/mac-gui`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-mac-gui.v${VERSION}.tar.bz2`" -print3ColsJustified "Windows GUI" "`getDirSizeMB $basedir/dist/win-gui`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-win-gui.v${VERSION}.zip`" -print3ColsJustified "Linux GUI" "`getDirSizeMB $basedir/dist/linux-gui`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-linux-gui.v${VERSION}.tar.bz2`" +print4ColsJustified "OS" "Uncompressed size (MB)" "Compressed size (MB)" "Reproducible build SHA-256" +print4ColsJustified "------------------------" "------------------------" "------------------------" "----------------------------------------------------------------" +print4ColsJustified "Mac" "`getFileSizeMB $basedir/dist/mac`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-mac.v${VERSION}.tar.bz2`" "`getHash $basedir/dist-zip/i2p-zero-mac.v${VERSION}.tar.bz2`" +print4ColsJustified "Windows" "`getFileSizeMB $basedir/dist/win`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-win.v${VERSION}.zip`" "`getHash $basedir/dist-zip/i2p-zero-win.v${VERSION}.zip`" +print4ColsJustified "Linux" "`getFileSizeMB $basedir/dist/linux`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-linux.v${VERSION}.tar.bz2`" "`getHash $basedir/dist-zip/i2p-zero-linux.v${VERSION}.tar.bz2`" +print4ColsJustified "Mac GUI" "`getFileSizeMB $basedir/dist/mac-gui`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-mac-gui.v${VERSION}.tar.bz2`" "`getHash $basedir/dist-zip/i2p-zero-mac-gui.v${VERSION}.tar.bz2`" +print4ColsJustified "Windows GUI" "`getFileSizeMB $basedir/dist/win-gui`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-win-gui.v${VERSION}.zip`" "`getHash $basedir/dist-zip/i2p-zero-win-gui.v${VERSION}.zip`" +print4ColsJustified "Linux GUI" "`getFileSizeMB $basedir/dist/linux-gui`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-linux-gui.v${VERSION}.tar.bz2`" "`getHash $basedir/dist-zip/i2p-zero-linux-gui.v${VERSION}.tar.bz2`" + +echo "" +echo "Note: Reproducible builds are a work in progress, since jlink is not reliably deterministic as of JDK12, particularly when building Windows distributables." \ No newline at end of file diff --git a/org.getmonero.util.normalizeZip/org.getmonero.util.normalizeZip.iml b/org.getmonero.util.normalizeZip/org.getmonero.util.normalizeZip.iml new file mode 100644 index 0000000..5731387 --- /dev/null +++ b/org.getmonero.util.normalizeZip/org.getmonero.util.normalizeZip.iml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/org.getmonero.util.normalizeZip/src/module-info.java b/org.getmonero.util.normalizeZip/src/module-info.java new file mode 100644 index 0000000..8825b25 --- /dev/null +++ b/org.getmonero.util.normalizeZip/src/module-info.java @@ -0,0 +1,3 @@ +module org.getmonero.util.normalizeZip { + requires org.apache.commons.compress; +} \ No newline at end of file diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/CompoundStripper.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/CompoundStripper.java new file mode 100644 index 0000000..64de975 --- /dev/null +++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/CompoundStripper.java @@ -0,0 +1,61 @@ +package org.getmonero.util.normalizeZip; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +/** + * A stripper that runs several strippers one after the others, + * where the input of one stripper is the output of the previous one. + * This class implements the Design Pattern "Decorator". + */ +final class CompoundStripper implements Stripper { + private final Stripper[] strippers; + + /** + * Constructs a compound stripper from a list of strippers. + * + * @param strippers the list of strippers. + */ + public CompoundStripper(Stripper... strippers) { + this.strippers = strippers; + } + + @Override + public void strip(File in, File out) throws IOException { + final List tmpFiles = new ArrayList<>(); + File currentIn = in; + try { + for (Stripper stripper : strippers) { + final File tmp = Files.createTempFile(null, null).toFile(); + tmp.deleteOnExit(); + tmpFiles.add(tmp); + stripper.strip(currentIn, tmp); + currentIn = tmp; + } + Files.copy(currentIn.toPath(), out.toPath(), StandardCopyOption.REPLACE_EXISTING); + } finally { + for (File file : tmpFiles) { + Files.delete(file.toPath()); + } + } + } + +} \ No newline at end of file diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/ManifestStripper.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/ManifestStripper.java new file mode 100644 index 0000000..4baad9b --- /dev/null +++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/ManifestStripper.java @@ -0,0 +1,67 @@ +package org.getmonero.util.normalizeZip; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Strips non-reproducible data from MANIFEST files. + * This stripper removes the following lines from the manifest: + * - Built-By + * - Created-By + * - Build-Jdk + * - Build-Date / Build-Time + * - Bnd-LastModified + * It also ensures that the MANIFEST entries are in a reproducible order + * (workaround for MSHARED-511 that was fixed in maven-archiver-3.0.1). + */ +public final class ManifestStripper implements Stripper { + private static final String[] DEFAULT_ATTRIBUTES = + {"Built-By", "Created-By", "Build-Jdk", "Build-Date", "Build-Time", + "Bnd-LastModified", "OpenIDE-Module-Build-Version"}; + private final List manifestAttributes; + + /** + * Creates a stripper that will remove a default list of manifest attributes. + */ + public ManifestStripper() { + this.manifestAttributes = new ArrayList(Arrays.asList(DEFAULT_ATTRIBUTES)); + } + + /** + * Creates a stripper that will remove a default list of manifest attributes + * plus additional user-specified attributes. + * + * @param manifestAttributes additional attributes to strip. + */ + public ManifestStripper(List manifestAttributes) { + this(); + if (manifestAttributes != null) { + this.manifestAttributes.addAll(manifestAttributes); + } + } + + @Override + public void strip(File in, File out) throws IOException { + final TextFileStripper s1 = new TextFileStripper(); + manifestAttributes.forEach(att -> s1.addPredicate(s -> s.startsWith(att + ":"))); + final SortManifestFileStripper s2 = new SortManifestFileStripper(); + new CompoundStripper(s1, s2).strip(in, out); + } +} \ No newline at end of file diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/NormalizeZip.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/NormalizeZip.java new file mode 100644 index 0000000..a1bf70c --- /dev/null +++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/NormalizeZip.java @@ -0,0 +1,199 @@ +package org.getmonero.util.normalizeZip;/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Based on: https://github.com/Zlika/reproducible-build-maven-plugin/blob/master/src/main/java/io/github/zlika/reproducible/ZipStripper.java + */ + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.FileTime; +import java.util.*; +import java.util.stream.Collectors; + +import org.apache.commons.compress.archivers.zip.X5455_ExtendedTimestamp; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.ZipExtraField; +import org.apache.commons.compress.archivers.zip.ZipFile; + +/** + * Strips non-reproducible data from a ZIP file. + * It rebuilds the ZIP file with a predictable order for the zip entries and sets zip entry dates to a fixed value. + */ +public final class NormalizeZip { + + public static void main(String[] args) throws Exception { + if(args.length!=2) { + System.out.println("Arguments: timestampSource zip"); + System.exit(1); + } + File timestampSourceFile = Path.of(args[0]).toFile(); + File sourceZipFile = Path.of(args[1]).toFile(); + File destZipFile = Path.of(args[1]+".tmp").toFile(); + + if(!timestampSourceFile.exists()) { + System.out.println("Timestamp source file does not exist"); + System.exit(1); + } + if(!sourceZipFile.exists()) { + System.out.println("Source zip file does not exist"); + System.exit(1); + } + + System.out.println("Normalizing zip " + sourceZipFile.getCanonicalPath() + " to timestamp: " + new Date(timestampSourceFile.lastModified())); + + NormalizeZip nz = new NormalizeZip(timestampSourceFile.lastModified(), true).addFileStripper("META-INF/MANIFEST.MF", new ManifestStripper()); + nz.strip(sourceZipFile, destZipFile); + if(!sourceZipFile.delete()) System.out.println("Cannot delete source file"); + if(!destZipFile.renameTo(sourceZipFile)) System.out.println("Cannot rename temporary zip file"); + destZipFile = sourceZipFile; + if(!destZipFile.setLastModified(timestampSourceFile.lastModified())) System.out.println("Cannot update zip last modified date"); + } + + /** + * Comparator used to sort the files in the ZIP file. + * This is mostly an alphabetical order comparator, with the exception that + * META-INF/MANIFEST.MF and META-INF/ must be the 2 first entries (if they exist) + * because this is required by some tools + * (cf. https://github.com/Zlika/reproducible-build-maven-plugin/issues/16). + */ + private static final Comparator MANIFEST_FILE_SORT_COMPARATOR = new Comparator() { + // CHECKSTYLE IGNORE LINE: ReturnCount + @Override + public int compare(String o1, String o2) { + if ("META-INF/MANIFEST.MF".equals(o1)) { + return -1; + } + if ("META-INF/MANIFEST.MF".equals(o2)) { + return 1; + } + if ("META-INF/".equals(o1)) { + return -1; + } + if ("META-INF/".equals(o2)) { + return 1; + } + return o1.compareTo(o2); + } + }; + + private final long zipTimestamp; + private final boolean fixZipExternalFileAttributes; + + + public NormalizeZip(long zipTimestamp, boolean fixZipExternalFileAttributes) { + this.zipTimestamp = zipTimestamp; + this.fixZipExternalFileAttributes = fixZipExternalFileAttributes; + } + + private final Map subFilters = new HashMap<>(); + private Stripper getSubFilter(String name) { + for (Map.Entry filter : subFilters.entrySet()) { + if (name.matches(filter.getKey())) { + return filter.getValue(); + } + } + return null; + } + /** + * Adds a stripper for a given file in the Zip. + * @param filename the name of the file in the Zip (regular expression). + * @param stripper the stripper to apply on the file. + * @return this object (for method chaining). + */ + public NormalizeZip addFileStripper(String filename, Stripper stripper) { + subFilters.put(filename, stripper); + return this; + } + + + public void strip(File in, File out) throws IOException { + try (final ZipFile zip = new ZipFile(in); + final ZipArchiveOutputStream zout = new ZipArchiveOutputStream(out)) { + final List sortedNames = sortEntriesByName(zip.getEntries()); + for (String name : sortedNames) { + final ZipArchiveEntry entry = zip.getEntry(name); + // Strip Zip entry + final ZipArchiveEntry strippedEntry = filterZipEntry(entry); + // Fix external file attributes if required + if (in.getName().endsWith(".jar") || in.getName().endsWith(".war")) { + fixAttributes(strippedEntry); + } + + // Strip file if required + final Stripper stripper = getSubFilter(name); + if (stripper != null) + { + // Unzip entry to temp file + final File tmp = File.createTempFile("tmp", null); + tmp.deleteOnExit(); + Files.copy(zip.getInputStream(entry), tmp.toPath(), StandardCopyOption.REPLACE_EXISTING); + final File tmp2 = File.createTempFile("tmp", null); + tmp2.deleteOnExit(); + stripper.strip(tmp, tmp2); + final byte[] fileContent = Files.readAllBytes(tmp2.toPath()); + strippedEntry.setSize(fileContent.length); + zout.putArchiveEntry(strippedEntry); + zout.write(fileContent); + zout.closeArchiveEntry(); + } + else { + // Copy the Zip entry as-is + zout.addRawArchiveEntry(strippedEntry, zip.getRawInputStream(entry)); + } + } + } + } + + private void fixAttributes(ZipArchiveEntry entry) { + if (fixZipExternalFileAttributes) { + /* ZIP external file attributes: + TTTTsstrwxrwxrwx0000000000ADVSHR + ^^^^____________________________ file type + (file: 1000 , dir: 0100) + ^^^_________________________ setuid, setgid, sticky + ^^^^^^^^^________________ Unix permissions + ^^^^^^ DOS attributes + The argument of setUnixMode() only takes the 2 upper bytes. */ + if (entry.isDirectory()) { + entry.setUnixMode((0b0100 << 12) + 0755); + } else { + entry.setUnixMode((0b1000 << 12) + 0644); + } + } + } + + private List sortEntriesByName(Enumeration entries) { + return Collections.list(entries).stream() + .map(e -> e.getName()) + .sorted(MANIFEST_FILE_SORT_COMPARATOR) + .collect(Collectors.toList()); + } + + private ZipArchiveEntry filterZipEntry(ZipArchiveEntry entry) { + // Set times + entry.setCreationTime(FileTime.fromMillis(zipTimestamp)); + entry.setLastAccessTime(FileTime.fromMillis(zipTimestamp)); + entry.setLastModifiedTime(FileTime.fromMillis(zipTimestamp)); + entry.setTime(zipTimestamp); + // Remove extended timestamps + for (ZipExtraField field : entry.getExtraFields()) { + if (field instanceof X5455_ExtendedTimestamp) { + entry.removeExtraField(field.getHeaderId()); + } + } + return entry; + } +} \ No newline at end of file diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/SortManifestFileStripper.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/SortManifestFileStripper.java new file mode 100644 index 0000000..02bee31 --- /dev/null +++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/SortManifestFileStripper.java @@ -0,0 +1,114 @@ +package org.getmonero.util.normalizeZip; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Sorts a MANIFEST file by attribute. + */ +final class SortManifestFileStripper implements Stripper { + private static final Comparator MANIFEST_ENTRY_COMPARATOR = new Comparator() { + // CHECKSTYLE IGNORE LINE: ReturnCount + @Override + public int compare(String o1, String o2) { + if (o1.startsWith("Manifest-Version:")) { + return -1; + } else if (o2.startsWith("Manifest-Version:")) { + return 1; + } + // From the "JAR File Specification": + // Each section must start with an attribute with the name as "Name" + else if (o1.startsWith("Name:") && !o2.startsWith("Name:")) { + return -1; + } else if (o2.startsWith("Name:") && !o1.startsWith("Name:")) { + return 1; + } else { + return o1.compareTo(o2); + } + } + }; + + @Override + public void strip(File in, File out) throws IOException { + final List lines = Files.readAllLines(in.toPath(), StandardCharsets.UTF_8); + + try (final BufferedWriter writer = new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(out), StandardCharsets.UTF_8))) { + final String sortedManifest = sortManifestSections(lines).stream() + .collect(Collectors.joining("\r\n")); + try { + writer.write(sortedManifest + "\r\n"); + } catch (IOException e) { + } + } + } + + private List sortManifestSections(List lines) { + final List> sections = new ArrayList<>(); + List currentSection = new ArrayList<>(); + for (String line : lines) { + // New section? + if (line.isEmpty()) { + if (!currentSection.isEmpty()) { + sections.add(currentSection); + currentSection = new ArrayList<>(); + } + } else { + currentSection.add(line); + } + } + if (!currentSection.isEmpty()) { + sections.add(currentSection); + } + + return sections.stream() + .map(list -> sortAttributes(list)) + .map(list -> String.join("", list)) + .sorted(MANIFEST_ENTRY_COMPARATOR) + .collect(Collectors.toList()); + } + + private List sortAttributes(List lines) { + final List attributes = new ArrayList<>(); + String currentAttribute = ""; + for (String line : lines) { + if (line.startsWith(" ")) { + currentAttribute += line + "\r\n"; + } else { + if (!currentAttribute.isEmpty()) { + attributes.add(currentAttribute); + currentAttribute = ""; + } + currentAttribute = line + "\r\n"; + } + } + if (!currentAttribute.isEmpty()) { + attributes.add(currentAttribute); + } + attributes.sort(MANIFEST_ENTRY_COMPARATOR); + return attributes; + } +} \ No newline at end of file diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/Stripper.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/Stripper.java new file mode 100644 index 0000000..f513b93 --- /dev/null +++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/Stripper.java @@ -0,0 +1,32 @@ +package org.getmonero.util.normalizeZip; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; + +/** + * Generic interface for stripping non-reproducible data. + */ +public interface Stripper +{ + /** + * Strips non-reproducible data. + * @param in the input file. + * @param out the stripped output file. + * @throws IOException if an I/O error occurs. + */ + void strip(File in, File out) throws IOException; +} \ No newline at end of file diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/TextFileStripper.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/TextFileStripper.java new file mode 100644 index 0000000..f428c2f --- /dev/null +++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/TextFileStripper.java @@ -0,0 +1,64 @@ +package org.getmonero.util.normalizeZip; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Predicate; + +/** + * Generic text file stripper. + */ +class TextFileStripper implements Stripper { + private final List> predicates = new ArrayList<>(); + + /** + * Adds a predicate to filter the text file. + * + * @param predicate the predicate. + * @return this. + */ + public TextFileStripper addPredicate(Predicate predicate) { + predicates.add(predicate.negate()); + return this; + } + + @Override + public void strip(File in, File out) throws IOException { + try (final BufferedWriter writer = new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(out), StandardCharsets.UTF_8)); + final BufferedReader reader = new BufferedReader( + new InputStreamReader(new FileInputStream(in), StandardCharsets.UTF_8))) { + reader.lines().filter(s -> predicates.stream().allMatch(p -> p.test(s))) + .forEach(s -> + { + try { + writer.write(s); + writer.write("\r\n"); + } catch (IOException e) { + } + }); + } + } +} \ No newline at end of file