diff --git a/bin/build-all-and-zip.sh b/bin/build-all-and-zip.sh
new file mode 100755
index 0000000..a217482
--- /dev/null
+++ b/bin/build-all-and-zip.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+if [ $(uname -s) = Darwin ]; then
+ basedir=$(dirname $(cd "$(dirname "$0")"; pwd -P))
+else
+ basedir=$(dirname $(dirname $(readlink -fm $0)))
+fi
+
+"$basedir"/bin/build-all.sh
+"$basedir"/bin/zip-all.sh
\ No newline at end of file
diff --git a/bin/build-all.sh b/bin/build-all.sh
index 10b7252..a7f98d2 100755
--- a/bin/build-all.sh
+++ b/bin/build-all.sh
@@ -11,6 +11,9 @@ rm -fr "$basedir/target" "$basedir/dist"
# retrieve the I2P Java sources, OpenJDK and the Ant build tool
"$basedir"/bin/import-packages.sh
+# build tool that normalizes zip/jar files for reproducible builds
+"$basedir"/bin/build-normalize-zip.sh
+
# build the i2p project retrieved from the I2P repository
"$basedir"/bin/build-original-i2p.sh
diff --git a/bin/build-launcher.sh b/bin/build-launcher.sh
index 2bb07f4..4d5abbc 100755
--- a/bin/build-launcher.sh
+++ b/bin/build-launcher.sh
@@ -7,7 +7,7 @@ else
fi
source "$basedir/bin/java-config.sh"
-
+source "$basedir/bin/util.sh"
echo "*** Compiling CLI"
"$JAVA_HOME"/bin/javac --module-path target/modules/combined.jar -d target/classes/org.getmonero.i2p.zero $(find org.getmonero.i2p.zero/src -name '*.java')
@@ -15,6 +15,7 @@ cp org.getmonero.i2p.zero/src/org/getmonero/i2p/zero/VERSION target/classes/org.
echo "*** Packaging CLI as a modular jar"
"$JAVA_HOME"/bin/jar --create --file target/org.getmonero.i2p.zero.jar --main-class org.getmonero.i2p.zero.Main -C target/classes/org.getmonero.i2p.zero .
+normalizeZip target/org.getmonero.i2p.zero.jar
echo "*** Compiling GUI"
"$JAVA_HOME"/bin/javac --module-path target/org.getmonero.i2p.zero.jar:target/modules/combined.jar:import/javafx-sdks/linux/javafx-sdk-$JAVAFX_VERSION/lib -d target/classes/org.getmonero.i2p.zero.gui $(find org.getmonero.i2p.zero.gui/src -name '*.java')
@@ -23,7 +24,7 @@ cp -r org.getmonero.i2p.zero.gui/src/org/getmonero/i2p/zero/gui/*.{css,png,fxml,
echo "*** Packaging GUI as a modular jar"
"$JAVA_HOME"/bin/jar --create --file target/org.getmonero.i2p.zero.gui.jar --main-class org.getmonero.i2p.zero.gui.Gui -C target/classes/org.getmonero.i2p.zero.gui .
-
+normalizeZip target/org.getmonero.i2p.zero.gui.jar
rm -fr "$basedir/dist"
for i in linux mac win linux-gui mac-gui win-gui; do mkdir -p "$basedir/dist/$i"; done
@@ -41,8 +42,8 @@ for i in linux mac win; do
JAVA_HOME_VARIANT=${JAVA_HOME_WIN} ;;
esac
echo "Using JAVA_HOME_VARIANT: $JAVA_HOME_VARIANT"
- "$JAVA_HOME"/bin/jlink --module-path "${JAVA_HOME_VARIANT}/jmods":target/modules:target/org.getmonero.i2p.zero.jar --add-modules combined,org.getmonero.i2p.zero --output dist/$i/router --compress 2 --no-header-files --no-man-pages
- "$JAVA_HOME"/bin/jlink --module-path "${JAVA_HOME_VARIANT}/jmods":import/javafx-jmods/$i/javafx-jmods-${JAVAFX_VERSION}:target/modules:target/org.getmonero.i2p.zero.jar:target/org.getmonero.i2p.zero.gui.jar --add-modules combined,org.getmonero.i2p.zero,org.getmonero.i2p.zero.gui,javafx.controls,javafx.fxml,java.desktop --output dist/$i-gui/router --compress 2 --no-header-files --no-man-pages
+ "$JAVA_HOME"/bin/jlink --module-path "${JAVA_HOME_VARIANT}/jmods":target/modules:target/org.getmonero.i2p.zero.jar --add-modules combined,org.getmonero.i2p.zero --output dist/$i/router --compress 2 --no-header-files --no-man-pages --order-resources=**/module-info.class,/java.base/java/lang/**,**javafx**
+ "$JAVA_HOME"/bin/jlink --module-path "${JAVA_HOME_VARIANT}/jmods":import/javafx-jmods/$i/javafx-jmods-${JAVAFX_VERSION}:target/modules:target/org.getmonero.i2p.zero.jar:target/org.getmonero.i2p.zero.gui.jar --add-modules combined,org.getmonero.i2p.zero,org.getmonero.i2p.zero.gui,javafx.controls,javafx.fxml,java.desktop --output dist/$i-gui/router --compress 2 --no-header-files --no-man-pages --order-resources=**/module-info.class,/java.base/java/lang/**,**javafx**
done
for i in linux mac linux-gui mac-gui; do
@@ -57,7 +58,6 @@ for i in linux-gui mac-gui; do
cp "$basedir/resources/launch-gui.sh" "$basedir/dist/$i/router/bin/"
done
-
for i in linux mac win linux-gui mac-gui win-gui; do cp -r "$basedir/import/i2p.base" "$basedir/dist/$i/router/"; done
# remove unnecessary native libs from jbigi.jar
@@ -67,11 +67,14 @@ for i in linux mac win; do
if [ "$j" = "mac" ]; then j="osx"; fi
if [ "$j" = "win" ]; then j="windows"; fi
zip -d "$basedir/dist/$i/router/i2p.base/jbigi.jar" *-${j}-*
- zip -d "$basedir/dist/$i-gui/router/i2p.base/jbigi.jar" *-${j}-*
+ normalizeZip "$basedir/dist/$i/router/i2p.base/jbigi.jar"
+ cp "$basedir/dist/$i/router/i2p.base/jbigi.jar" "$basedir/dist/$i-gui/router/i2p.base/jbigi.jar"
fi
done
done
+
+
# build mac gui app structure
mv "$basedir/dist/mac-gui/router" "$basedir/dist/mac-gui/router-tmp"
mkdir -p "$basedir/dist/mac-gui/router/i2p-zero.app/Contents/MacOS/"
@@ -121,6 +124,8 @@ du -sk dist/* | awk '{printf "%.1f MB %s\n",$1/1024,$2}'
echo "*** Done ***"
+echo "To build the distribution archives and show reproducible build SHA-256 hashes, type: bin/zip-all.sh"
+echo ""
os_name=`uname -s`
if [ $os_name = Darwin ]; then
os_name=mac
diff --git a/bin/build-normalize-zip.sh b/bin/build-normalize-zip.sh
new file mode 100755
index 0000000..c017ae7
--- /dev/null
+++ b/bin/build-normalize-zip.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ $(uname -s) = Darwin ]; then
+ basedir=$(dirname $(cd "$(dirname "$0")"; pwd -P))
+else
+ basedir=$(dirname $(dirname $(readlink -fm $0)))
+fi
+
+source "$basedir/bin/java-config.sh"
+source "$basedir/bin/util.sh"
+
+echo "*** Compiling Zip normalizer utility"
+"$JAVA_HOME"/bin/javac --module-path import/commons-compress-1.18/commons-compress-1.18.jar -d target/classes/org.getmonero.util.normalizeZip $(find org.getmonero.util.normalizeZip/src -name '*.java')
+
+echo "*** Packaging Zip normalizer as a modular jar"
+"$JAVA_HOME"/bin/jar --create --file target/org.getmonero.util.normalizeZip.jar --main-class org.getmonero.util.normalizeZip.NormalizeZip -C target/classes/org.getmonero.util.normalizeZip .
+
+
diff --git a/bin/clean-build-env.sh b/bin/clean-build-env.sh
new file mode 100755
index 0000000..92aa25c
--- /dev/null
+++ b/bin/clean-build-env.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ $(uname -s) = Darwin ]; then
+ basedir=$(dirname $(cd "$(dirname "$0")"; pwd -P))
+else
+ basedir=$(dirname $(dirname $(readlink -fm $0)))
+fi
+
+rm -fr "$basedir/dist-zip" "$basedir/dist-zip-staging" "$basedir/dist" "$basedir/target" "$basedir/import" "$basedir/out"
\ No newline at end of file
diff --git a/bin/convert-jars-to-modules.sh b/bin/convert-jars-to-modules.sh
index c82eb6e..ca264b5 100755
--- a/bin/convert-jars-to-modules.sh
+++ b/bin/convert-jars-to-modules.sh
@@ -7,6 +7,7 @@ else
fi
source "$basedir/bin/java-config.sh"
+source "$basedir/bin/util.sh"
cp "$basedir"/import/jetty-lib/*.jar "$basedir/import/lib/"
@@ -49,4 +50,6 @@ echo "*** Creating new combined modular jar"
cp $combinedJarPath "$basedir/target/modules/"
"$JAVA_HOME"/bin/jar uf "$basedir/target/modules/combined.jar" -C "$basedir/target/module-info/combined" module-info.class
+normalizeZip target/modules/combined.jar
+
diff --git a/bin/import-packages.sh b/bin/import-packages.sh
index f540153..657aca7 100755
--- a/bin/import-packages.sh
+++ b/bin/import-packages.sh
@@ -37,6 +37,11 @@ if [ ! -d "$basedir/import/apache-ant-1.10.5" ]; then
tar zxvf apache-ant-1.10.5-bin.tar.gz
fi
+if [ ! -d "$basedir/import/commons-compress-1.18" ]; then
+ wget https://www.mirrorservice.org/sites/ftp.apache.org//commons/compress/binaries/commons-compress-1.18-bin.tar.gz
+ tar zxvf commons-compress-1.18-bin.tar.gz
+fi
+
if [ ! -d "$basedir/import/jetty-lib" ]; then
mkdir -p jetty-lib
wget --directory-prefix=jetty-lib http://central.maven.org/maven2/org/eclipse/jetty/jetty-server/9.4.14.v20181114/jetty-server-9.4.14.v20181114.jar
diff --git a/bin/util.sh b/bin/util.sh
new file mode 100644
index 0000000..166d8dd
--- /dev/null
+++ b/bin/util.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# get the SHA-256 hash of the specified file
+getHash () {
+ if [ $(uname -s) = Darwin ]; then
+ h=`shasum -a 256 $1 | awk '{print $1}'`
+ else
+ h=`sha256sum $1 | awk '{print $1}'`
+ fi
+ echo $h
+}
+
+# normalizes the specified jar or zip for reproducible build. Enforces consistent zip file order and sets all timestamps to the last modified date of the VERSION file
+normalizeZip () {
+ java --module-path "$basedir/import/commons-compress-1.18/commons-compress-1.18.jar":"$basedir/target/org.getmonero.util.normalizeZip.jar" \
+ -m org.getmonero.util.normalizeZip "$basedir/org.getmonero.i2p.zero/src/org/getmonero/i2p/zero/VERSION" "$1"
+}
\ No newline at end of file
diff --git a/bin/zip-all.sh b/bin/zip-all.sh
index 7336356..d6e7049 100755
--- a/bin/zip-all.sh
+++ b/bin/zip-all.sh
@@ -6,18 +6,24 @@ else
basedir=$(dirname $(dirname $(readlink -fm $0)))
fi
+source "$basedir/bin/util.sh"
+
+VERSION=$(head -n 1 "$basedir/org.getmonero.i2p.zero/src/org/getmonero/i2p/zero/VERSION")
+
rm -fr "$basedir/dist-zip-staging"
mkdir -p "$basedir/dist-zip-staging"
rm -fr "$basedir/dist-zip"
mkdir -p "$basedir/dist-zip"
-VERSION=$(head -n 1 "$basedir/org.getmonero.i2p.zero/src/org/getmonero/i2p/zero/VERSION")
-
cd "$basedir/dist"
for i in linux linux-gui mac mac-gui win win-gui; do cp -r ${i} "$basedir"/dist-zip-staging/i2p-zero-${i}.v${VERSION}; done
+versionDate=`date -r "$basedir"/org.getmonero.i2p.zero/src/org/getmonero/i2p/zero/VERSION +"%Y%m%d%H%M.%S"`
+
+find "$basedir"/dist-zip-staging -exec touch -t $versionDate {} \;
+
cd "$basedir/dist-zip-staging"
for i in win win-gui; do zip -r9 "$basedir"/dist-zip/i2p-zero-${i}.v${VERSION}.zip i2p-zero-${i}.v${VERSION}; done
@@ -31,21 +37,26 @@ fi
cd ..
-print3ColsJustified () {
- printf "%-.14s %s" "$1 " "| "
- printf "%-.26s %s" "$2 " "| "
- printf "%s\n" "$3"
+print4ColsJustified () {
+ printf "%-.12s %s" "$1 " "| "
+ printf "%-.23s %s" "$2 " "| "
+ printf "%-.21s %s" "$3 " "| "
+ printf "%s\n" "$4"
}
-getDirSizeMB () {
- du -sk $1 | awk '{printf "%.1f",$1/1024,$2}'
+getFileSizeMB () {
+ s=`du -sk $1 | awk '{printf "%.1f",$1/1024,$2}'`
+ echo $s
}
-print3ColsJustified "OS" "Uncompressed size (MB)" "Compressed size (MB)"
-print3ColsJustified "------------------------" "------------------------" "------------------------"
-print3ColsJustified "Mac" "`getDirSizeMB $basedir/dist/mac`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-mac.v${VERSION}.tar.bz2`"
-print3ColsJustified "Windows" "`getDirSizeMB $basedir/dist/win`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-win.v${VERSION}.zip`"
-print3ColsJustified "Linux" "`getDirSizeMB $basedir/dist/linux`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-linux.v${VERSION}.tar.bz2`"
-print3ColsJustified "Mac GUI" "`getDirSizeMB $basedir/dist/mac-gui`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-mac-gui.v${VERSION}.tar.bz2`"
-print3ColsJustified "Windows GUI" "`getDirSizeMB $basedir/dist/win-gui`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-win-gui.v${VERSION}.zip`"
-print3ColsJustified "Linux GUI" "`getDirSizeMB $basedir/dist/linux-gui`" "`getDirSizeMB $basedir/dist-zip/i2p-zero-linux-gui.v${VERSION}.tar.bz2`"
+print4ColsJustified "OS" "Uncompressed size (MB)" "Compressed size (MB)" "Reproducible build SHA-256"
+print4ColsJustified "------------------------" "------------------------" "------------------------" "----------------------------------------------------------------"
+print4ColsJustified "Mac" "`getFileSizeMB $basedir/dist/mac`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-mac.v${VERSION}.tar.bz2`" "`getHash $basedir/dist-zip/i2p-zero-mac.v${VERSION}.tar.bz2`"
+print4ColsJustified "Windows" "`getFileSizeMB $basedir/dist/win`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-win.v${VERSION}.zip`" "`getHash $basedir/dist-zip/i2p-zero-win.v${VERSION}.zip`"
+print4ColsJustified "Linux" "`getFileSizeMB $basedir/dist/linux`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-linux.v${VERSION}.tar.bz2`" "`getHash $basedir/dist-zip/i2p-zero-linux.v${VERSION}.tar.bz2`"
+print4ColsJustified "Mac GUI" "`getFileSizeMB $basedir/dist/mac-gui`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-mac-gui.v${VERSION}.tar.bz2`" "`getHash $basedir/dist-zip/i2p-zero-mac-gui.v${VERSION}.tar.bz2`"
+print4ColsJustified "Windows GUI" "`getFileSizeMB $basedir/dist/win-gui`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-win-gui.v${VERSION}.zip`" "`getHash $basedir/dist-zip/i2p-zero-win-gui.v${VERSION}.zip`"
+print4ColsJustified "Linux GUI" "`getFileSizeMB $basedir/dist/linux-gui`" "`getFileSizeMB $basedir/dist-zip/i2p-zero-linux-gui.v${VERSION}.tar.bz2`" "`getHash $basedir/dist-zip/i2p-zero-linux-gui.v${VERSION}.tar.bz2`"
+
+echo ""
+echo "Note: Reproducible builds are a work in progress, since jlink is not reliably deterministic as of JDK12, particularly when building Windows distributables."
\ No newline at end of file
diff --git a/org.getmonero.util.normalizeZip/org.getmonero.util.normalizeZip.iml b/org.getmonero.util.normalizeZip/org.getmonero.util.normalizeZip.iml
new file mode 100644
index 0000000..5731387
--- /dev/null
+++ b/org.getmonero.util.normalizeZip/org.getmonero.util.normalizeZip.iml
@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/org.getmonero.util.normalizeZip/src/module-info.java b/org.getmonero.util.normalizeZip/src/module-info.java
new file mode 100644
index 0000000..8825b25
--- /dev/null
+++ b/org.getmonero.util.normalizeZip/src/module-info.java
@@ -0,0 +1,3 @@
+module org.getmonero.util.normalizeZip {
+ requires org.apache.commons.compress;
+}
\ No newline at end of file
diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/CompoundStripper.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/CompoundStripper.java
new file mode 100644
index 0000000..64de975
--- /dev/null
+++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/CompoundStripper.java
@@ -0,0 +1,61 @@
+package org.getmonero.util.normalizeZip;
+
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A stripper that runs several strippers one after the others,
+ * where the input of one stripper is the output of the previous one.
+ * This class implements the Design Pattern "Decorator".
+ */
+final class CompoundStripper implements Stripper {
+ private final Stripper[] strippers;
+
+ /**
+ * Constructs a compound stripper from a list of strippers.
+ *
+ * @param strippers the list of strippers.
+ */
+ public CompoundStripper(Stripper... strippers) {
+ this.strippers = strippers;
+ }
+
+ @Override
+ public void strip(File in, File out) throws IOException {
+ final List tmpFiles = new ArrayList<>();
+ File currentIn = in;
+ try {
+ for (Stripper stripper : strippers) {
+ final File tmp = Files.createTempFile(null, null).toFile();
+ tmp.deleteOnExit();
+ tmpFiles.add(tmp);
+ stripper.strip(currentIn, tmp);
+ currentIn = tmp;
+ }
+ Files.copy(currentIn.toPath(), out.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ } finally {
+ for (File file : tmpFiles) {
+ Files.delete(file.toPath());
+ }
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/ManifestStripper.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/ManifestStripper.java
new file mode 100644
index 0000000..4baad9b
--- /dev/null
+++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/ManifestStripper.java
@@ -0,0 +1,67 @@
+package org.getmonero.util.normalizeZip;
+
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Strips non-reproducible data from MANIFEST files.
+ * This stripper removes the following lines from the manifest:
+ * - Built-By
+ * - Created-By
+ * - Build-Jdk
+ * - Build-Date / Build-Time
+ * - Bnd-LastModified
+ * It also ensures that the MANIFEST entries are in a reproducible order
+ * (workaround for MSHARED-511 that was fixed in maven-archiver-3.0.1).
+ */
+public final class ManifestStripper implements Stripper {
+ private static final String[] DEFAULT_ATTRIBUTES =
+ {"Built-By", "Created-By", "Build-Jdk", "Build-Date", "Build-Time",
+ "Bnd-LastModified", "OpenIDE-Module-Build-Version"};
+ private final List manifestAttributes;
+
+ /**
+ * Creates a stripper that will remove a default list of manifest attributes.
+ */
+ public ManifestStripper() {
+ this.manifestAttributes = new ArrayList(Arrays.asList(DEFAULT_ATTRIBUTES));
+ }
+
+ /**
+ * Creates a stripper that will remove a default list of manifest attributes
+ * plus additional user-specified attributes.
+ *
+ * @param manifestAttributes additional attributes to strip.
+ */
+ public ManifestStripper(List manifestAttributes) {
+ this();
+ if (manifestAttributes != null) {
+ this.manifestAttributes.addAll(manifestAttributes);
+ }
+ }
+
+ @Override
+ public void strip(File in, File out) throws IOException {
+ final TextFileStripper s1 = new TextFileStripper();
+ manifestAttributes.forEach(att -> s1.addPredicate(s -> s.startsWith(att + ":")));
+ final SortManifestFileStripper s2 = new SortManifestFileStripper();
+ new CompoundStripper(s1, s2).strip(in, out);
+ }
+}
\ No newline at end of file
diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/NormalizeZip.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/NormalizeZip.java
new file mode 100644
index 0000000..a1bf70c
--- /dev/null
+++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/NormalizeZip.java
@@ -0,0 +1,199 @@
+package org.getmonero.util.normalizeZip;/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Based on: https://github.com/Zlika/reproducible-build-maven-plugin/blob/master/src/main/java/io/github/zlika/reproducible/ZipStripper.java
+ */
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.FileTime;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.commons.compress.archivers.zip.X5455_ExtendedTimestamp;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipExtraField;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+
+/**
+ * Strips non-reproducible data from a ZIP file.
+ * It rebuilds the ZIP file with a predictable order for the zip entries and sets zip entry dates to a fixed value.
+ */
+public final class NormalizeZip {
+
+ public static void main(String[] args) throws Exception {
+ if(args.length!=2) {
+ System.out.println("Arguments: timestampSource zip");
+ System.exit(1);
+ }
+ File timestampSourceFile = Path.of(args[0]).toFile();
+ File sourceZipFile = Path.of(args[1]).toFile();
+ File destZipFile = Path.of(args[1]+".tmp").toFile();
+
+ if(!timestampSourceFile.exists()) {
+ System.out.println("Timestamp source file does not exist");
+ System.exit(1);
+ }
+ if(!sourceZipFile.exists()) {
+ System.out.println("Source zip file does not exist");
+ System.exit(1);
+ }
+
+ System.out.println("Normalizing zip " + sourceZipFile.getCanonicalPath() + " to timestamp: " + new Date(timestampSourceFile.lastModified()));
+
+ NormalizeZip nz = new NormalizeZip(timestampSourceFile.lastModified(), true).addFileStripper("META-INF/MANIFEST.MF", new ManifestStripper());
+ nz.strip(sourceZipFile, destZipFile);
+ if(!sourceZipFile.delete()) System.out.println("Cannot delete source file");
+ if(!destZipFile.renameTo(sourceZipFile)) System.out.println("Cannot rename temporary zip file");
+ destZipFile = sourceZipFile;
+ if(!destZipFile.setLastModified(timestampSourceFile.lastModified())) System.out.println("Cannot update zip last modified date");
+ }
+
+ /**
+ * Comparator used to sort the files in the ZIP file.
+ * This is mostly an alphabetical order comparator, with the exception that
+ * META-INF/MANIFEST.MF and META-INF/ must be the 2 first entries (if they exist)
+ * because this is required by some tools
+ * (cf. https://github.com/Zlika/reproducible-build-maven-plugin/issues/16).
+ */
+ private static final Comparator MANIFEST_FILE_SORT_COMPARATOR = new Comparator() {
+ // CHECKSTYLE IGNORE LINE: ReturnCount
+ @Override
+ public int compare(String o1, String o2) {
+ if ("META-INF/MANIFEST.MF".equals(o1)) {
+ return -1;
+ }
+ if ("META-INF/MANIFEST.MF".equals(o2)) {
+ return 1;
+ }
+ if ("META-INF/".equals(o1)) {
+ return -1;
+ }
+ if ("META-INF/".equals(o2)) {
+ return 1;
+ }
+ return o1.compareTo(o2);
+ }
+ };
+
+ private final long zipTimestamp;
+ private final boolean fixZipExternalFileAttributes;
+
+
+ public NormalizeZip(long zipTimestamp, boolean fixZipExternalFileAttributes) {
+ this.zipTimestamp = zipTimestamp;
+ this.fixZipExternalFileAttributes = fixZipExternalFileAttributes;
+ }
+
+ private final Map subFilters = new HashMap<>();
+ private Stripper getSubFilter(String name) {
+ for (Map.Entry filter : subFilters.entrySet()) {
+ if (name.matches(filter.getKey())) {
+ return filter.getValue();
+ }
+ }
+ return null;
+ }
+ /**
+ * Adds a stripper for a given file in the Zip.
+ * @param filename the name of the file in the Zip (regular expression).
+ * @param stripper the stripper to apply on the file.
+ * @return this object (for method chaining).
+ */
+ public NormalizeZip addFileStripper(String filename, Stripper stripper) {
+ subFilters.put(filename, stripper);
+ return this;
+ }
+
+
+ public void strip(File in, File out) throws IOException {
+ try (final ZipFile zip = new ZipFile(in);
+ final ZipArchiveOutputStream zout = new ZipArchiveOutputStream(out)) {
+ final List sortedNames = sortEntriesByName(zip.getEntries());
+ for (String name : sortedNames) {
+ final ZipArchiveEntry entry = zip.getEntry(name);
+ // Strip Zip entry
+ final ZipArchiveEntry strippedEntry = filterZipEntry(entry);
+ // Fix external file attributes if required
+ if (in.getName().endsWith(".jar") || in.getName().endsWith(".war")) {
+ fixAttributes(strippedEntry);
+ }
+
+ // Strip file if required
+ final Stripper stripper = getSubFilter(name);
+ if (stripper != null)
+ {
+ // Unzip entry to temp file
+ final File tmp = File.createTempFile("tmp", null);
+ tmp.deleteOnExit();
+ Files.copy(zip.getInputStream(entry), tmp.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ final File tmp2 = File.createTempFile("tmp", null);
+ tmp2.deleteOnExit();
+ stripper.strip(tmp, tmp2);
+ final byte[] fileContent = Files.readAllBytes(tmp2.toPath());
+ strippedEntry.setSize(fileContent.length);
+ zout.putArchiveEntry(strippedEntry);
+ zout.write(fileContent);
+ zout.closeArchiveEntry();
+ }
+ else {
+ // Copy the Zip entry as-is
+ zout.addRawArchiveEntry(strippedEntry, zip.getRawInputStream(entry));
+ }
+ }
+ }
+ }
+
+ private void fixAttributes(ZipArchiveEntry entry) {
+ if (fixZipExternalFileAttributes) {
+ /* ZIP external file attributes:
+ TTTTsstrwxrwxrwx0000000000ADVSHR
+ ^^^^____________________________ file type
+ (file: 1000 , dir: 0100)
+ ^^^_________________________ setuid, setgid, sticky
+ ^^^^^^^^^________________ Unix permissions
+ ^^^^^^ DOS attributes
+ The argument of setUnixMode() only takes the 2 upper bytes. */
+ if (entry.isDirectory()) {
+ entry.setUnixMode((0b0100 << 12) + 0755);
+ } else {
+ entry.setUnixMode((0b1000 << 12) + 0644);
+ }
+ }
+ }
+
+ private List sortEntriesByName(Enumeration entries) {
+ return Collections.list(entries).stream()
+ .map(e -> e.getName())
+ .sorted(MANIFEST_FILE_SORT_COMPARATOR)
+ .collect(Collectors.toList());
+ }
+
+ private ZipArchiveEntry filterZipEntry(ZipArchiveEntry entry) {
+ // Set times
+ entry.setCreationTime(FileTime.fromMillis(zipTimestamp));
+ entry.setLastAccessTime(FileTime.fromMillis(zipTimestamp));
+ entry.setLastModifiedTime(FileTime.fromMillis(zipTimestamp));
+ entry.setTime(zipTimestamp);
+ // Remove extended timestamps
+ for (ZipExtraField field : entry.getExtraFields()) {
+ if (field instanceof X5455_ExtendedTimestamp) {
+ entry.removeExtraField(field.getHeaderId());
+ }
+ }
+ return entry;
+ }
+}
\ No newline at end of file
diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/SortManifestFileStripper.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/SortManifestFileStripper.java
new file mode 100644
index 0000000..02bee31
--- /dev/null
+++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/SortManifestFileStripper.java
@@ -0,0 +1,114 @@
+package org.getmonero.util.normalizeZip;
+
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Sorts a MANIFEST file by attribute.
+ */
+final class SortManifestFileStripper implements Stripper {
+ private static final Comparator MANIFEST_ENTRY_COMPARATOR = new Comparator() {
+ // CHECKSTYLE IGNORE LINE: ReturnCount
+ @Override
+ public int compare(String o1, String o2) {
+ if (o1.startsWith("Manifest-Version:")) {
+ return -1;
+ } else if (o2.startsWith("Manifest-Version:")) {
+ return 1;
+ }
+ // From the "JAR File Specification":
+ // Each section must start with an attribute with the name as "Name"
+ else if (o1.startsWith("Name:") && !o2.startsWith("Name:")) {
+ return -1;
+ } else if (o2.startsWith("Name:") && !o1.startsWith("Name:")) {
+ return 1;
+ } else {
+ return o1.compareTo(o2);
+ }
+ }
+ };
+
+ @Override
+ public void strip(File in, File out) throws IOException {
+ final List lines = Files.readAllLines(in.toPath(), StandardCharsets.UTF_8);
+
+ try (final BufferedWriter writer = new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(out), StandardCharsets.UTF_8))) {
+ final String sortedManifest = sortManifestSections(lines).stream()
+ .collect(Collectors.joining("\r\n"));
+ try {
+ writer.write(sortedManifest + "\r\n");
+ } catch (IOException e) {
+ }
+ }
+ }
+
+ private List sortManifestSections(List lines) {
+ final List> sections = new ArrayList<>();
+ List currentSection = new ArrayList<>();
+ for (String line : lines) {
+ // New section?
+ if (line.isEmpty()) {
+ if (!currentSection.isEmpty()) {
+ sections.add(currentSection);
+ currentSection = new ArrayList<>();
+ }
+ } else {
+ currentSection.add(line);
+ }
+ }
+ if (!currentSection.isEmpty()) {
+ sections.add(currentSection);
+ }
+
+ return sections.stream()
+ .map(list -> sortAttributes(list))
+ .map(list -> String.join("", list))
+ .sorted(MANIFEST_ENTRY_COMPARATOR)
+ .collect(Collectors.toList());
+ }
+
+ private List sortAttributes(List lines) {
+ final List attributes = new ArrayList<>();
+ String currentAttribute = "";
+ for (String line : lines) {
+ if (line.startsWith(" ")) {
+ currentAttribute += line + "\r\n";
+ } else {
+ if (!currentAttribute.isEmpty()) {
+ attributes.add(currentAttribute);
+ currentAttribute = "";
+ }
+ currentAttribute = line + "\r\n";
+ }
+ }
+ if (!currentAttribute.isEmpty()) {
+ attributes.add(currentAttribute);
+ }
+ attributes.sort(MANIFEST_ENTRY_COMPARATOR);
+ return attributes;
+ }
+}
\ No newline at end of file
diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/Stripper.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/Stripper.java
new file mode 100644
index 0000000..f513b93
--- /dev/null
+++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/Stripper.java
@@ -0,0 +1,32 @@
+package org.getmonero.util.normalizeZip;
+
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Generic interface for stripping non-reproducible data.
+ */
+public interface Stripper
+{
+ /**
+ * Strips non-reproducible data.
+ * @param in the input file.
+ * @param out the stripped output file.
+ * @throws IOException if an I/O error occurs.
+ */
+ void strip(File in, File out) throws IOException;
+}
\ No newline at end of file
diff --git a/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/TextFileStripper.java b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/TextFileStripper.java
new file mode 100644
index 0000000..f428c2f
--- /dev/null
+++ b/org.getmonero.util.normalizeZip/src/org/getmonero/util/normalizeZip/TextFileStripper.java
@@ -0,0 +1,64 @@
+package org.getmonero.util.normalizeZip;
+
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Predicate;
+
+/**
+ * Generic text file stripper.
+ */
+class TextFileStripper implements Stripper {
+ private final List> predicates = new ArrayList<>();
+
+ /**
+ * Adds a predicate to filter the text file.
+ *
+ * @param predicate the predicate.
+ * @return this.
+ */
+ public TextFileStripper addPredicate(Predicate predicate) {
+ predicates.add(predicate.negate());
+ return this;
+ }
+
+ @Override
+ public void strip(File in, File out) throws IOException {
+ try (final BufferedWriter writer = new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(out), StandardCharsets.UTF_8));
+ final BufferedReader reader = new BufferedReader(
+ new InputStreamReader(new FileInputStream(in), StandardCharsets.UTF_8))) {
+ reader.lines().filter(s -> predicates.stream().allMatch(p -> p.test(s)))
+ .forEach(s ->
+ {
+ try {
+ writer.write(s);
+ writer.write("\r\n");
+ } catch (IOException e) {
+ }
+ });
+ }
+ }
+}
\ No newline at end of file