[neon/backports-jammy/debuerreotype/Neon/unstable] /: 0.14-1 (patches unapplied)
git-ubuntu importer
null at kde.org
Fri Aug 11 03:02:00 BST 2023
Git commit 11c96b41d86865b9e5692693c81b10a9b3eedacb by git-ubuntu importer, on behalf of Tianon Gravi.
Committed on 22/03/2022 at 23:28.
Pushed by carlosdem into branch 'Neon/unstable'.
0.14-1 (patches unapplied)
Imported using git-ubuntu import.
A +11 -0 .docker-image.sh
M +1 -0 .dockerignore
A +72 -0 .github/workflows/ci.yml
D +0 -40 .travis.yml
R +7 -4 .validate-debian.sh [from: .travis.sh - 066% similarity]
A +24 -0 .validate-ubuntu.sh
M +46 -13 Dockerfile
M +8 -5 README.md
M +1 -1 VERSION
D +0 -73 build-all.sh
D +0 -339 build.sh
M +20 -0 debian/changelog
M +1 -1 debian/tests/control
M +13 -4 debian/tests/stretch
A +82 -0 docker-run.sh
A +164 -0 examples/debian-all.sh
A +363 -0 examples/debian.sh
A +279 -0 examples/oci-image.sh
A +198 -0 examples/raspbian.sh
A +184 -0 examples/steamos.sh
A +194 -0 examples/ubuntu.sh
D +0 -177 raspbian.sh
M +20 -6 scripts/.apt-version.sh
M +1 -1 scripts/.constants.sh
A +99 -0 scripts/.debian-mirror.sh
A +124 -0 scripts/.debootstrap-scripts/potato
A +101 -0 scripts/.debootstrap-scripts/slink
M +10 -1 scripts/.dpkg-arch.sh
M +1 -1 scripts/.fix-apt-comments.sh
M +2 -1 scripts/.gpgv-ignore-expiration.sh
M +4 -4 scripts/.slimify-excludes
M +5 -5 scripts/.slimify-includes
M +1 -1 scripts/.snapshot-url.sh
M +1 -1 scripts/debuerreotype-apt-get
M +5 -3 scripts/debuerreotype-chroot
M +41 -34 scripts/debuerreotype-debian-sources-list
M +8 -1 scripts/debuerreotype-fixup
M +1 -1 scripts/debuerreotype-gpgv-ignore-expiration-config
M +59 -35 scripts/debuerreotype-init
M +5 -3 scripts/debuerreotype-minimizing-config
A +39 -0 scripts/debuerreotype-recalculate-epoch
M +20 -1 scripts/debuerreotype-slimify
M +1 -1 scripts/debuerreotype-tar
M +1 -1 scripts/debuerreotype-version
D +0 -188 steamos.sh
D +0 -192 ubuntu.sh
https://invent.kde.org/neon/backports-jammy/debuerreotype/-/commit/11c96b41d86865b9e5692693c81b10a9b3eedacb
diff --git a/.docker-image.sh b/.docker-image.sh
new file mode 100755
index 0000000..c21efa1
--- /dev/null
+++ b/.docker-image.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+thisDir="$(readlink -vf "$BASH_SOURCE")"
+thisDir="$(dirname "$thisDir")"
+
+ver="$("$thisDir/scripts/debuerreotype-version")"
+ver="${ver%% *}"
+dockerImage="debuerreotype/debuerreotype:$ver"
+
+echo "$dockerImage"
diff --git a/.dockerignore b/.dockerignore
index c33e173..ecaa375 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,4 +1,5 @@
**
!VERSION
+!examples/
!scripts/
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..eebe2a7
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,72 @@
+name: GitHub CI
+
+on:
+ pull_request:
+ push:
+ schedule:
+ - cron: 0 0 * * 0
+
+defaults:
+ run:
+ shell: 'bash -Eeuo pipefail -x {0}'
+
+jobs:
+
+ https:
+ name: Ensure no-TLS snapshot usage
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout at v2
+ - name: Ensure http://snapshot.debian.org (https://github.com/debuerreotype/debuerreotype/pull/119#issuecomment-901457009)
+ run: |
+ rm .github/workflows/ci.yml # this file itself will always be a match, but it's the only valid one 👀
+ ! grep -rn 'https://snapshot.debian.org'
+
+ test:
+ strategy:
+ matrix:
+ include:
+ - { SUITE: stable, CODENAME: jessie, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: a712fb9b228f107a0d65e836127a377857ab70388f2e23d3b583ac56f8f5a75b }
+ - { SUITE: jessie, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: a712fb9b228f107a0d65e836127a377857ab70388f2e23d3b583ac56f8f5a75b }
+ - { SUITE: testing, CODENAME: stretch, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 56adf91c1aca0b52ba7c818f4cdc93e47a6c2c93266296d7e0e108ddead825b4 }
+ - { SUITE: stretch, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 56adf91c1aca0b52ba7c818f4cdc93e47a6c2c93266296d7e0e108ddead825b4 }
+ - { SUITE: unstable, CODENAME: sid, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 87f46eeb98d44ff5742d87112d9cc45e51dbb1204d60cb4136b51f0edfce061f }
+ - { SUITE: sid, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 87f46eeb98d44ff5742d87112d9cc45e51dbb1204d60cb4136b51f0edfce061f }
+ - { SUITE: oldstable, CODENAME: wheezy, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 52f942d1d0f5ed2d41cb81b63091dc5da0872da94b80a684937cdab7fc57ac06 }
+ - { SUITE: wheezy, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 52f942d1d0f5ed2d41cb81b63091dc5da0872da94b80a684937cdab7fc57ac06 }
+
+ # EOL suites testing
+ - { SUITE: eol, CODENAME: etch, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: 893d436a060f2536f70efbdfd2e2952cf311eada558f858e6190c80b323b783e }
+ - { SUITE: eol, CODENAME: lenny, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: c263084bc482b1538512eb091095dd30cf55a6873b989aeee9d4e148f2f3fafa }
+ - { SUITE: eol, CODENAME: woody, ARCH: i386, TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: f80833896e141fbfebf8c91e79da2ccca1bdeb8f8ecc4e05dd33531c32857e0f }
+ - { SUITE: eol, CODENAME: jessie, TIMESTAMP: "2021-03-01T00:00:00Z", SHA256: a151dd2b4209e152c9323778e3d1016b26e89881df7dd3d81a39282cbd37bdae }
+
+ # qemu-debootstrap testing
+ - { ARCH: arm64, SUITE: jessie, CODENAME: "", TIMESTAMP: "2017-01-01T00:00:00Z", SHA256: e0bbe759a5df8a3201c3bfc984a91d7ee46bb7cb939d7c06275ff374043ed731 }
+ - { ARCH: sh4, SUITE: unstable, CODENAME: "", TIMESTAMP: "2022-02-01T00:00:00Z", SHA256: 82949d1099c18dd13bd2d732238d94ae70d03abc50afc6950baee257cce37fd1 }
+ - { ARCH: riscv64, SUITE: unstable, CODENAME: "", TIMESTAMP: "2022-02-01T00:00:00Z", SHA256: 2a4accea7aef15d4979435ec16f898dcb784330b7b4991fc9ca38b11f87035f5 }
+
+ # a few entries for "today" to try and catch issues like https://github.com/debuerreotype/debuerreotype/issues/41 sooner
+ - { SUITE: unstable, CODENAME: "", TIMESTAMP: "today 00:00:00", SHA256: "" }
+ - { SUITE: stable, CODENAME: "", TIMESTAMP: "today 00:00:00", SHA256: "" }
+ - { SUITE: oldstable, CODENAME: "", TIMESTAMP: "today 00:00:00", SHA256: "" }
+
+ - { DISTRO: ubuntu, SUITE: bionic }
+ - { DISTRO: ubuntu, SUITE: focal }
+ fail-fast: false
+ name: Test ${{ matrix.DISTRO && format('{0} ', matrix.DISTRO) }}${{ matrix.SUITE }}${{ matrix.CODENAME && format(' ({0})', matrix.CODENAME) }}${{ matrix.ARCH && format(' [{0}]', matrix.ARCH) }}${{ matrix.TIMESTAMP && format(' at {0}', matrix.TIMESTAMP) }}
+ runs-on: ubuntu-20.04
+ env: ${{ matrix }}
+ steps:
+ - uses: actions/checkout at v2
+ - name: Prepare Environment
+ run: |
+ sudo apt-get update -qq
+ sudo apt-get install -yqq binfmt-support qemu-user-static
+ docker run -d --name squignix --restart always tianon/squignix
+ git clone --depth 1 https://github.com/tianon/pgp-happy-eyeballs.git ~/phe
+ ~/phe/hack-my-builds.sh
+ rm -rf ~/phe
+ - name: Build
+ run: |
+ "./.validate-${DISTRO:-debian}.sh"
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index c8e7102..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-language: bash
-services: docker
-
-env:
- - SUITE=stable CODENAME=jessie TIMESTAMP=2017-01-01T00:00:00Z SHA256=55ba54fdca819df18d813be36503b0a02abf1570c3bf5999b10891ccca5448e2
- - SUITE=jessie CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=55ba54fdca819df18d813be36503b0a02abf1570c3bf5999b10891ccca5448e2
- - SUITE=testing CODENAME=stretch TIMESTAMP=2017-01-01T00:00:00Z SHA256=1608c820c1d9c9d8adf210f80b1d751e5c26179aa27a1c1ddb8e41ae0222d8c4
- - SUITE=stretch CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=1608c820c1d9c9d8adf210f80b1d751e5c26179aa27a1c1ddb8e41ae0222d8c4
- - SUITE=unstable CODENAME=sid TIMESTAMP=2017-01-01T00:00:00Z SHA256=49a5152822ec9f0e1a61ff1d02671681f12fc1aba083f39e972f6ff897b69c80
- - SUITE=sid CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=49a5152822ec9f0e1a61ff1d02671681f12fc1aba083f39e972f6ff897b69c80
- - SUITE=oldstable CODENAME=wheezy TIMESTAMP=2017-01-01T00:00:00Z SHA256=f1bd72548e3c25ce222fb9e2bb57a5b6d4b01042180894fb05d83a0251e6dab1
- - SUITE=wheezy CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=f1bd72548e3c25ce222fb9e2bb57a5b6d4b01042180894fb05d83a0251e6dab1
- # EOL suites testing
- - SUITE=eol CODENAME=etch TIMESTAMP=2017-01-01T00:00:00Z SHA256=b48e999ab4fda1720b0dc863d38cdd4d6b55530f34f262a28949eb6173102da9
- - SUITE=eol CODENAME=lenny TIMESTAMP=2017-01-01T00:00:00Z SHA256=1a2fffd34daa4a6bb968aebe86480a4093035a23700ec5f2e883423b9b4dcfa7
- - SUITE=eol CODENAME=woody ARCH=i386 TIMESTAMP=2017-01-01T00:00:00Z SHA256=ef4bc81e31db51fa9f095811ddbcc8a005f05f098596317d5a138fa90157bf40
- # qemu-debootstrap testing
- - ARCH=arm64 SUITE=jessie CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=893efc1b9db1ba2df4f171d4422194a408f9810d3b55d9b0cd66fcc7722f7567
- # a few entries for "today" to try and catch issues like https://github.com/debuerreotype/debuerreotype/issues/41 sooner
- - SUITE=unstable CODENAME= TIMESTAMP="today 00:00:00" SHA256=
- - SUITE=stable CODENAME= TIMESTAMP="today 00:00:00" SHA256=
- - SUITE=oldstable CODENAME= TIMESTAMP="today 00:00:00" SHA256=
-
-addons:
- apt:
- packages:
- - binfmt-support
- - qemu-user-static
-
-before_script:
- - docker run -d --name squignix --restart always tianon/squignix # TODO temporary!! (once https://github.com/tianon/pgp-happy-eyeballs/tree/travis-squignix is deleted, this should be) -- squignix is necessary for building etch and woody who otherwise are so poorly behaved they get rate limited by snapshot.d.o (https://travis-ci.org/debuerreotype/debuerreotype/builds/479633791)
- - wget -qO- https://github.com/tianon/pgp-happy-eyeballs/raw/713f2a81bf3eac1752f0c41b271444f5b57e93c9/hack-my-builds.sh | bash
-
-script:
- - travis_retry ./.travis.sh
-
-after_script:
- - docker images
- - docker logs rawdns
- - docker logs squignix # TODO temporary!! (see above)
diff --git a/.travis.sh b/.validate-debian.sh
similarity index 66%
rename from .travis.sh
rename to .validate-debian.sh
index 903988f..a631c0b 100755
--- a/.travis.sh
+++ b/.validate-debian.sh
@@ -14,17 +14,20 @@ fi
if [ -n "${ARCH:-}" ]; then
buildArgs+=( "--arch=${ARCH}" )
if [ "$ARCH" != 'i386' ]; then
- buildArgs+=( '--qemu' )
+ if [ "$ARCH" != 'arm64' ]; then
+ buildArgs+=( '--ports' )
+ fi
fi
fi
-buildArgs+=( travis "$SUITE" "@$epoch" )
+buildArgs+=( validate "$SUITE" "@$epoch" )
-checkFile="travis/$serial/${ARCH:-amd64}/${CODENAME:-$SUITE}/rootfs.tar.xz"
+checkFile="validate/$serial/${ARCH:-amd64}/${CODENAME:-$SUITE}/rootfs.tar.xz"
+mkdir -p validate
set -x
./scripts/debuerreotype-version
-./build.sh "${buildArgs[@]}"
+./docker-run.sh --pull ./examples/debian.sh "${buildArgs[@]}"
real="$(sha256sum "$checkFile" | cut -d' ' -f1)"
[ -z "$SHA256" ] || [ "$SHA256" = "$real" ]
diff --git a/.validate-ubuntu.sh b/.validate-ubuntu.sh
new file mode 100755
index 0000000..5c4302c
--- /dev/null
+++ b/.validate-ubuntu.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+dockerImage="$(./.docker-image.sh)"
+dockerImage+='-ubuntu'
+{
+ cat Dockerfile - <<-'EODF'
+ RUN set -eux; \
+# https://bugs.debian.org/929165 :(
+# http://snapshot.debian.org/package/ubuntu-keyring/
+# http://snapshot.debian.org/package/ubuntu-keyring/2020.06.17.1-1/
+ wget -O ubuntu-keyring.deb 'http://snapshot.debian.org/archive/debian/20210307T083530Z/pool/main/u/ubuntu-keyring/ubuntu-keyring_2020.06.17.1-1_all.deb'; \
+ echo 'c2d8c4a9be6244bbea80c2e0e7624cbd3a2006a2 *ubuntu-keyring.deb' | sha1sum --strict --check -; \
+ apt-get install -y --no-install-recommends ./ubuntu-keyring.deb; \
+ rm ubuntu-keyring.deb
+ EODF
+} | docker build --pull --tag "$dockerImage" --file - .
+
+mkdir -p validate
+
+set -x
+
+./scripts/debuerreotype-version
+./docker-run.sh --image="$dockerImage" --no-build ./examples/ubuntu.sh validate "$SUITE"
diff --git a/Dockerfile b/Dockerfile
index 8c42313..ff30de9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,27 +1,58 @@
# docker run --cap-add SYS_ADMIN --cap-drop SETFCAP --tmpfs /tmp:dev,exec,suid,noatime ...
# bootstrapping a new architecture?
-# ./scripts/debuerreotype-init /tmp/docker-rootfs stretch now
+# ./scripts/debuerreotype-init /tmp/docker-rootfs bullseye now
# ./scripts/debuerreotype-minimizing-config /tmp/docker-rootfs
-# ./scripts/debuerreotype-debian-sources-list /tmp/docker-rootfs stretch
-# ./scripts/debuerreotype-tar /tmp/docker-rootfs - | docker import - debian:stretch-slim
+# ./scripts/debuerreotype-debian-sources-list /tmp/docker-rootfs bullseye
+# ./scripts/debuerreotype-tar /tmp/docker-rootfs - | docker import - debian:bullseye-slim
# alternate:
-# debootstrap --variant=minbase stretch /tmp/docker-rootfs
-# tar -cC /tmp/docker-rootfs . | docker import - debian:stretch-slim
+# debootstrap --variant=minbase bullseye /tmp/docker-rootfs
+# tar -cC /tmp/docker-rootfs . | docker import - debian:bullseye-slim
# (or your own favorite set of "debootstrap" commands to create a base image for building this one FROM)
-FROM debian:stretch-slim
+FROM debian:bullseye-slim
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN set -eux; \
+ apt-get update; \
+ apt-get install -y --no-install-recommends \
+ debian-ports-archive-keyring \
debootstrap \
wget ca-certificates \
xz-utils \
\
gnupg dirmngr \
- && rm -rf /var/lib/apt/lists/*
+ ; \
+ rm -rf /var/lib/apt/lists/*
+
+# fight the tyrrany of HSTS (which destroys our ability to transparently cache snapshot.debian.org responses)
+ENV WGETRC /.wgetrc
+RUN echo 'hsts=0' >> "$WGETRC"
+
+# https://github.com/debuerreotype/debuerreotype/issues/100
+# https://tracker.debian.org/pkg/distro-info-data
+# http://snapshot.debian.org/package/distro-info-data/
+# http://snapshot.debian.org/package/distro-info-data/0.51/
+RUN set -eux; \
+ wget -O distro-info-data.deb 'http://snapshot.debian.org/archive/debian/20210724T033023Z/pool/main/d/distro-info-data/distro-info-data_0.51_all.deb'; \
+ echo 'c5f4a3bd999d3d79612dfec285e4afc4f6248648 *distro-info-data.deb' | sha1sum --strict --check -; \
+ apt-get install -y ./distro-info-data.deb; \
+ rm distro-info-data.deb; \
+ [ -s /usr/share/distro-info/debian.csv ]
+
+# https://bugs.debian.org/973852
+# https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/63
+# https://people.debian.org/~tianon/debootstrap-mr-63--download_main.patch
+RUN set -eux; \
+ apt-get update; \
+ apt-get install -y --no-install-recommends patch; \
+ rm -rf /var/lib/apt/lists/*; \
+ wget -O debootstrap-download-main.patch 'https://people.debian.org/~tianon/debootstrap-mr-63--download_main.patch'; \
+ echo 'ceae8f508a9b49236fa4519a44a584e6c774aa0e4446eb1551f3b69874a4cde5 *debootstrap-download-main.patch' | sha256sum --strict --check -; \
+ patch --input=debootstrap-download-main.patch /usr/share/debootstrap/functions; \
+ rm debootstrap-download-main.patch
# see ".dockerignore"
COPY . /opt/debuerreotype
-RUN set -ex; \
+RUN set -eux; \
cd /opt/debuerreotype/scripts; \
for f in debuerreotype-*; do \
ln -svL "$PWD/$f" "/usr/local/bin/$f"; \
@@ -34,12 +65,14 @@ WORKDIR /tmp
# a few example md5sum values for amd64:
-# debuerreotype-init test-stretch stretch 2017-05-08T00:00:00Z
+# debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg test-stretch stretch 2017-05-08T00:00:00Z
# debuerreotype-tar test-stretch test-stretch.tar
# md5sum test-stretch.tar
-# 14206d5b9b2991e98f5214c3d310e4fa
+# 694f02c53651673ebe094cae3bcbb06d
+# ./docker-run.sh sh -euxc 'debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg /tmp/rootfs stretch 2017-05-08T00:00:00Z; debuerreotype-tar /tmp/rootfs - | md5sum'
-# debuerreotype-init test-jessie jessie 2017-05-08T00:00:00Z
+# debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg test-jessie jessie 2017-05-08T00:00:00Z
# debuerreotype-tar test-jessie test-jessie.tar
# md5sum test-jessie.tar
-# 57f98d3636000630080e5ba208508e10
+# 354cedd99c08d213d3493a7cf0aaaad6
+# ./docker-run.sh sh -euxc 'debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg /tmp/rootfs jessie 2017-05-08T00:00:00Z; debuerreotype-tar /tmp/rootfs - | md5sum'
diff --git a/README.md b/README.md
index 46ab8d2..4f59dec 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
# Debuerreotype
-[](https://travis-ci.org/debuerreotype/debuerreotype/branches)
+[](https://github.com/debuerreotype/debuerreotype/actions?query=workflow%3A%22GitHub+CI%22+branch%3Amaster)
Reproducible, [snapshot](http://snapshot.debian.org)-based Debian rootfs builds (especially for Docker).
@@ -33,6 +33,7 @@ Available scripts:
| `debuerreotype-minimizing-config` | apply configuration tweaks to make the rootfs minimal and keep it minimal (especially targeted at Docker images, with comments explicitly describing Docker use cases) |
| `debuerreotype-slimify` | remove files such as documentation to create an even smaller rootfs (used for creating `slim` variants of the Docker images, for example) |
| `debuerreotype-debian-sources-list` | generate an appropriate Debian `sources.list` in the rootfs given a suite (especially for updating `sources.list` to point at deb.debian.org before generating outputs) |
+| `debuerreotype-recalculate-epoch` | (esp. for non-Debian) recalculate `debuerreotype-epoch` from `/var/lib/apt/lists/*_{In,}Release` files' `Date:` fields (after updating `sources.list` / `apt-get update`) |
| `debuerreotype-fixup` | invoked by `debuerreotype-tar` to fixup timestamps and remove known-bad log files for determinism |
| `debuerreotype-tar` | deterministically create a tar file of the rootfs |
| `debuerreotype-version` | print out the version of the current `debuerreotype` installation |
@@ -41,10 +42,10 @@ A simple `Dockerfile` is provided for using these scripts in a simple determinis
The provided `Dockerfile` also includes comments with hints for bootstrapping the environment on a new architecture (which then presumably doesn't have a `debian` Docker base image yet).
-Full example: (see [`build.sh`](build.sh) for this in practice)
+Full example: (see [`examples/debian.sh`](examples/debian.sh) for this in practice)
```console
-$ debuerreotype-init rootfs stretch 2017-01-01T00:00:00Z
+$ debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg rootfs stretch 2017-01-01T00:00:00Z
I: Retrieving InRelease
I: Checking Release signature
I: Valid Release signature (key id 126C0D24BD8A2942CC7DF8AC7638D0442B90D010)
@@ -89,15 +90,17 @@ Processing triggers for libc-bin (2.24-8) ...
$ debuerreotype-debian-sources-list rootfs stretch
$ debuerreotype-tar rootfs - | sha256sum
-a076d4cd04f68ee117e598a40cc947ad051fc8b063340da015fdceddeb1b0e75 -
+e6f10da22f7ab5996f855c85ad5ae38cd786029c57893436c3bb2320f30bc188 -
$ # try it! you should get that same sha256sum value!
```
+(As a one-liner via [`docker-run.sh`](docker-run.sh): `./docker-run.sh sh -euxc 'debuerreotype-init --keyring /usr/share/keyrings/debian-archive-removed-keys.gpg /tmp/rootfs stretch 2017-01-01T00:00:00Z; debuerreotype-minimizing-config /tmp/rootfs; debuerreotype-apt-get /tmp/rootfs update -qq; debuerreotype-apt-get /tmp/rootfs dist-upgrade -yqq; debuerreotype-apt-get /tmp/rootfs install -yqq --no-install-recommends inetutils-ping iproute2; debuerreotype-debian-sources-list /tmp/rootfs stretch; debuerreotype-tar /tmp/rootfs - | sha256sum'`)
+
## How much have you verified this?
Well, I ran the scripts across seven explicit architectures (`amd64`, `arm64`, `armel`, `armhf`, `i386`, `ppc64el`, `s390x`) and eight explicit suites (`oldstable`, `stable`, `testing`, `unstable`, `wheezy`, `jessie`, `stretch`, `sid`) for a timestamp of `2017-05-16T00:00:00Z` (where supported, since `wheezy`/`oldstable` didn't or no longer currently supports some of those architectures), and there were no modifications to any of the tarballs after several runs across several days.
-Additionally, Travis runs with a fixed timestamp value across several suites to verify that their checksums are reproducible, as expected.
+Additionally, GitHub Actions runs with a fixed timestamp value across several suites to verify that their checksums are reproducible, as expected.
From time to time, comments in the files generated by `debuerreotype-minimizing-config` might change (for example), which would obviously result in a different checksum, but a simple [`diffoscope`](https://diffoscope.org/) should be sufficient to verify that the change is benign.
diff --git a/VERSION b/VERSION
index 68c123c..948a547 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.10
+0.14
diff --git a/build-all.sh b/build-all.sh
deleted file mode 100755
index eb5d300..0000000
--- a/build-all.sh
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env bash
-set -Eeuo pipefail
-
-suites=(
- unstable
- testing
- stable
- oldstable
- oldoldstable
-
- # just in case (will no-op with "not supported on 'arch'" unless it exists)
- oldoldoldstable
-)
-
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
-source "$thisDir/scripts/.constants.sh" \
- --flags 'no-build' \
- -- \
- '[--no-build] <output-dir> <timestamp>' \
- 'output 2017-05-08T00:00:00Z'
-
-eval "$dgetopt"
-build=1
-while true; do
- flag="$1"; shift
- dgetopt-case "$flag"
- case "$flag" in
- --no-build) build= ;; # for skipping "docker build"
- --) break ;;
- *) eusage "unknown flag '$flag'" ;;
- esac
-done
-
-outputDir="${1:-}"; shift || eusage 'missing output-dir'
-timestamp="${1:-}"; shift || eusage 'missing timestamp'
-
-mkdir -p "$outputDir"
-outputDir="$(readlink -f "$outputDir")"
-
-ver="$("$thisDir/scripts/debuerreotype-version")"
-ver="${ver%% *}"
-dockerImage="debuerreotype/debuerreotype:$ver"
-[ -z "$build" ] || docker build -t "$dockerImage" "$thisDir"
-
-mirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp")"
-secmirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp" 'debian-security')"
-
-dpkgArch="$(docker run --rm "$dockerImage" dpkg --print-architecture | awk -F- '{ print $NF }')"
-echo
-echo "-- BUILDING TARBALLS FOR '$dpkgArch' FROM '$mirror/' --"
-echo
-
-for suite in "${suites[@]}"; do
- doSkip=
- case "$suite" in
- testing|unstable) ;;
- *)
- if ! wget --quiet --spider "$secmirror/dists/$suite/updates/main/binary-$dpkgArch/Packages.gz"; then
- doSkip=1
- fi
- ;;
- esac
- if ! wget --quiet --spider "$mirror/dists/$suite/main/binary-$dpkgArch/Packages.gz"; then
- doSkip=1
- fi
- if [ -n "$doSkip" ]; then
- echo >&2
- echo >&2 "warning: '$suite' not supported on '$dpkgArch' (at '$timestamp'); skipping"
- echo >&2
- continue
- fi
- "$thisDir/build.sh" --no-build --codename-copy "$outputDir" "$suite" "$timestamp"
-done
diff --git a/build.sh b/build.sh
deleted file mode 100755
index 425475a..0000000
--- a/build.sh
+++ /dev/null
@@ -1,339 +0,0 @@
-#!/usr/bin/env bash
-set -Eeuo pipefail
-
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
-source "$thisDir/scripts/.constants.sh" \
- --flags 'no-build,codename-copy' \
- --flags 'eol,arch:,qemu' \
- -- \
- '[--no-build] [--codename-copy] [--eol] [--arch=<arch>] [--qemu] <output-dir> <suite> <timestamp>' \
- 'output stretch 2017-05-08T00:00:00Z
---codename-copy output stable 2017-05-08T00:00:00Z
---eol output squeeze 2016-03-14T00:00:00Z
---eol --arch i386 output sarge 2016-03-14T00:00:00Z' \
-
-eval "$dgetopt"
-build=1
-codenameCopy=
-eol=
-arch=
-qemu=
-while true; do
- flag="$1"; shift
- dgetopt-case "$flag"
- case "$flag" in
- --no-build) build= ;; # for skipping "docker build"
- --codename-copy) codenameCopy=1 ;; # for copying a "stable.tar.xz" to "stretch.tar.xz" with updated sources.list (saves a lot of extra building work)
- --eol) eol=1 ;; # for using "archive.debian.org"
- --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init
- --qemu) qemu=1 ;; # for using "qemu-debootstrap"
- --) break ;;
- *) eusage "unknown flag '$flag'" ;;
- esac
-done
-
-outputDir="${1:-}"; shift || eusage 'missing output-dir'
-suite="${1:-}"; shift || eusage 'missing suite'
-timestamp="${1:-}"; shift || eusage 'missing timestamp'
-
-mkdir -p "$outputDir"
-outputDir="$(readlink -f "$outputDir")"
-
-securityArgs=(
- --cap-add SYS_ADMIN
- --cap-drop SETFCAP
-)
-if docker info | grep -q apparmor; then
- # AppArmor blocks mount :)
- securityArgs+=(
- --security-opt apparmor=unconfined
- )
-fi
-
-if [ "$suite" = 'potato' ]; then
- # --debian-eol potato wants to run "chroot ... mount ... /proc" which gets blocked (i386, ancient binaries, blah blah blah)
- securityArgs+=(
- --security-opt seccomp=unconfined
- )
-fi
-
-ver="$("$thisDir/scripts/debuerreotype-version")"
-ver="${ver%% *}"
-dockerImage="debuerreotype/debuerreotype:$ver"
-[ -z "$build" ] || docker build -t "$dockerImage" "$thisDir"
-if [ -n "$qemu" ]; then
- [ -z "$build" ] || docker build -t "$dockerImage-qemu" - <<-EODF
- FROM $dockerImage
- RUN apt-get update && apt-get install -y --no-install-recommends qemu-user-static && rm -rf /var/lib/apt/lists/*
- EODF
- dockerImage="$dockerImage-qemu"
-fi
-
-docker run \
- --rm \
- "${securityArgs[@]}" \
- --tmpfs /tmp:dev,exec,suid,noatime \
- -w /tmp \
- -e suite="$suite" \
- -e timestamp="$timestamp" \
- -e codenameCopy="$codenameCopy" \
- -e eol="$eol" -e arch="$arch" -e qemu="$qemu" \
- -e TZ='UTC' -e LC_ALL='C' \
- --hostname debuerreotype \
- "$dockerImage" \
- bash -Eeuo pipefail -c '
- set -x
-
- epoch="$(date --date "$timestamp" +%s)"
- serial="$(date --date "@$epoch" +%Y%m%d)"
- dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- "{ print \$NF }")}"
-
- exportDir="output"
- outputDir="$exportDir/$serial/$dpkgArch/$suite"
-
- touch_epoch() {
- while [ "$#" -gt 0 ]; do
- local f="$1"; shift
- touch --no-dereference --date="@$epoch" "$f"
- done
- }
-
- debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")"
-
- for archive in "" security; do
- if [ -z "$eol" ]; then
- snapshotUrl="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "@$epoch" "${archive:+debian-${archive}}")"
- else
- snapshotUrl="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "@$epoch" "debian-archive")/debian${archive:+-${archive}}"
- fi
- snapshotUrlFile="$exportDir/$serial/$dpkgArch/snapshot-url${archive:+-${archive}}"
- mkdir -p "$(dirname "$snapshotUrlFile")"
- echo "$snapshotUrl" > "$snapshotUrlFile"
- touch_epoch "$snapshotUrlFile"
- done
-
- export GNUPGHOME="$(mktemp -d)"
- keyring="$GNUPGHOME/debian-archive-$suite-keyring.gpg"
- if [ "$suite" = potato ]; then
- # src:debian-archive-keyring was created in 2006, thus does not include a key for potato
- gpg --batch --no-default-keyring --keyring "$keyring" \
- --keyserver ha.pool.sks-keyservers.net \
- --recv-keys 8FD47FF1AA9372C37043DC28AA7DEB7B722F1AED
- else
- # check against all releases (ie, combine both "debian-archive-keyring.gpg" and "debian-archive-removed-keys.gpg"), since we cannot really know whether the target release became EOL later than the snapshot date we are targeting
- gpg --batch --no-default-keyring --keyring "$keyring" --import \
- /usr/share/keyrings/debian-archive-keyring.gpg \
- /usr/share/keyrings/debian-archive-removed-keys.gpg
- fi
-
- snapshotUrl="$(< "$exportDir/$serial/$dpkgArch/snapshot-url")"
- mkdir -p "$outputDir"
- wget -O "$outputDir/Release.gpg" "$snapshotUrl/dists/$suite/Release.gpg"
- wget -O "$outputDir/Release" "$snapshotUrl/dists/$suite/Release"
- gpgv \
- --keyring "$keyring" \
- "$outputDir/Release.gpg" \
- "$outputDir/Release"
-
- codename="$(awk -F ": " "\$1 == \"Codename\" { print \$2; exit }" "$outputDir/Release")"
- if [ -n "$codenameCopy" ] && [ "$codename" = "$suite" ]; then
- # if codename already is the same as suite, then making a copy does not make any sense
- codenameCopy=
- fi
- if [ -n "$codenameCopy" ] && [ -z "$codename" ]; then
- echo >&2 "error: --codename-copy specified but we failed to get a Codename for $suite"
- exit 1
- fi
-
- {
- initArgs=( --arch="$dpkgArch" )
- if [ -z "$eol" ]; then
- initArgs+=( --debian )
- else
- initArgs+=( --debian-eol )
- fi
- initArgs+=( --keyring "$keyring" )
-
- # disable merged-usr (for now?) due to the following compelling arguments:
- # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc)
- # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still)
- # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406
- initArgs+=( --no-merged-usr )
-
- if [ -n "$qemu" ]; then
- initArgs+=( --debootstrap="qemu-debootstrap" )
- fi
-
- debuerreotype-init "${initArgs[@]}" rootfs "$suite" "@$epoch"
-
- if [ -n "$eol" ]; then
- debuerreotype-gpgv-ignore-expiration-config rootfs
- fi
-
- debuerreotype-minimizing-config rootfs
- debuerreotype-apt-get rootfs update -qq
- debuerreotype-apt-get rootfs dist-upgrade -yqq
-
- aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" rootfs)"
- if dpkg --compare-versions "$aptVersion" ">=" "0.7.14~"; then
- # https://salsa.debian.org/apt-team/apt/commit/06d79436542ccf3e9664306da05ba4c34fba4882
- noInstallRecommends="--no-install-recommends"
- else
- # --debian-eol etch and lower do not support --no-install-recommends
- noInstallRecommends="-o APT::Install-Recommends=0"
- fi
-
- if [ -n "$eol" ] && dpkg --compare-versions "$aptVersion" ">=" "0.7.26~"; then
- # https://salsa.debian.org/apt-team/apt/commit/1ddb859611d2e0f3d9ea12085001810f689e8c99
- echo "Acquire::Check-Valid-Until \"false\";" > rootfs/etc/apt/apt.conf.d/check-valid-until.conf
- # TODO make this a real script so it can have a nice comment explaining why we do it for EOL releases?
- fi
-
- # make a couple copies of rootfs so we can create other variants
- for variant in slim sbuild; do
- mkdir "rootfs-$variant"
- tar -cC rootfs . | tar -xC "rootfs-$variant"
- done
-
- # prefer iproute2 if it exists
- iproute=iproute2
- if ! debuerreotype-apt-get rootfs install -qq -s iproute2 &> /dev/null; then
- # poor wheezy
- iproute=iproute
- fi
- ping=iputils-ping
- if debuerreotype-chroot rootfs bash -c "command -v ping > /dev/null"; then
- # if we already have "ping" (as in --debian-eol potato), skip installing any extra ping package
- ping=
- fi
- debuerreotype-apt-get rootfs install -y $noInstallRecommends $ping $iproute
-
- debuerreotype-slimify rootfs-slim
-
- # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild
- # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26
- # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194
- debuerreotype-apt-get rootfs-sbuild install -y $noInstallRecommends build-essential fakeroot
-
- create_artifacts() {
- local targetBase="$1"; shift
- local rootfs="$1"; shift
- local suite="$1"; shift
- local variant="$1"; shift
-
- # make a copy of the snapshot-facing sources.list file before we overwrite it
- cp "$rootfs/etc/apt/sources.list" "$targetBase.sources-list-snapshot"
- touch_epoch "$targetBase.sources-list-snapshot"
-
- local tarArgs=()
- if [ -n "$qemu" ]; then
- tarArgs+=( --exclude="./usr/bin/qemu-*-static" )
- fi
-
- if [ "$variant" != "sbuild" ]; then
- debuerreotype-debian-sources-list $([ -z "$eol" ] || echo "--eol") "$rootfs" "$suite"
- else
- # sbuild needs "deb-src" entries
- debuerreotype-debian-sources-list --deb-src $([ -z "$eol" ] || echo "--eol") "$rootfs" "$suite"
-
- # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes
- # (which are used in sbuild for "--extra-package")
- # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
- # ...
- # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
- rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes"
- # TODO figure out the bug and fix it in APT instead /o\
-
- # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar")
- # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521
- tarArgs+=( --include-dev )
- fi
-
- case "$suite" in
- sarge)
- # for some reason, sarge creates "/var/cache/man/index.db" with some obvious embedded unix timestamps (but if we exclude it, "man" still works properly, so *shrug*)
- tarArgs+=( --exclude ./var/cache/man/index.db )
- ;;
-
- woody)
- # woody not only contains "exim", but launches it during our build process and tries to email "root at debuerreotype" (which fails and creates non-reproducibility)
- tarArgs+=( --exclude ./var/spool/exim --exclude ./var/log/exim )
- ;;
-
- potato)
- tarArgs+=(
- # for some reason, pototo leaves a core dump (TODO figure out why??)
- --exclude "./core"
- --exclude "./qemu*.core"
- # also, it leaves some junk in /tmp (/tmp/fdmount.conf.tmp.XXX)
- --exclude "./tmp/fdmount.conf.tmp.*"
- )
- ;;
- esac
-
- debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz"
- du -hsx "$targetBase.tar.xz"
-
- sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256"
- touch_epoch "$targetBase.tar.xz.sha256"
-
- debuerreotype-chroot "$rootfs" bash -c "
- if ! dpkg-query -W 2> /dev/null; then
- # --debian-eol woody has no dpkg-query
- dpkg -l
- fi
- " > "$targetBase.manifest"
- echo "$epoch" > "$targetBase.debuerreotype-epoch"
- debuerreotype-version > "$targetBase.debuerreotype-version"
- touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch" "$targetBase.debuerreotype-version"
-
- for f in debian_version os-release apt/sources.list; do
- targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")"
- if [ -e "$rootfs/etc/$f" ]; then
- # /etc/os-release does not exist in --debian-eol squeeze, for example (hence the existence check)
- cp "$rootfs/etc/$f" "$targetFile"
- touch_epoch "$targetFile"
- fi
- done
- }
-
- for rootfs in rootfs*/; do
- rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ...
-
- du -hsx "$rootfs"
-
- variant="${rootfs#rootfs}" # "", "-slim", ...
- variant="${variant#-}" # "", "slim", ...
-
- variantDir="$outputDir/$variant"
- mkdir -p "$variantDir"
-
- targetBase="$variantDir/rootfs"
-
- create_artifacts "$targetBase" "$rootfs" "$suite" "$variant"
- done
-
- if [ -n "$codenameCopy" ]; then
- codenameDir="$exportDir/$serial/$dpkgArch/$codename"
- mkdir -p "$codenameDir"
- tar -cC "$outputDir" --exclude="**/rootfs.*" . | tar -xC "$codenameDir"
-
- for rootfs in rootfs*/; do
- rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ...
-
- variant="${rootfs#rootfs}" # "", "-slim", ...
- variant="${variant#-}" # "", "slim", ...
-
- variantDir="$codenameDir/$variant"
- targetBase="$variantDir/rootfs"
-
- # point sources.list back at snapshot.debian.org temporarily (but this time pointing at $codename instead of $suite)
- debuerreotype-debian-sources-list --snapshot $([ -z "$eol" ] || echo "--eol") "$rootfs" "$codename"
-
- create_artifacts "$targetBase" "$rootfs" "$codename" "$variant"
- done
- fi
- } >&2
-
- tar -cC "$exportDir" .
- ' | tar -xvC "$outputDir"
diff --git a/debian/changelog b/debian/changelog
index d6a707b..ec5445d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,23 @@
+debuerreotype (0.14-1) unstable; urgency=medium
+
+ [ Tianon Gravi ]
+ * Add several more architecture hashes to autopkgtests:
+ - armel
+ - armhf
+ - i386
+ - mips64el
+ - ppc64el
+ - s390x
+
+ [ Gianfranco Costamagna ]
+ * Depend on debian-archive-keyring for autopkgtests (Closes: #985963)
+
+ [ Tianon Gravi ]
+ * Update to 0.14 upstream release
+ - most notable change is more correct "/etc/machine-id" handling
+
+ -- Tianon Gravi <tianon at debian.org> Tue, 22 Mar 2022 09:36:10 -0700
+
debuerreotype (0.10-2) unstable; urgency=medium
[ Tianon Gravi ]
diff --git a/debian/tests/control b/debian/tests/control
index a5273a8..4653eaa 100644
--- a/debian/tests/control
+++ b/debian/tests/control
@@ -1,5 +1,6 @@
Tests: stretch
Depends: ca-certificates,
+ debian-archive-keyring,
debootstrap,
debuerreotype,
diffoscope,
@@ -7,4 +8,3 @@ Depends: ca-certificates,
wget,
xz-utils
Restrictions: allow-stderr, needs-root
-#Architecture: amd64, arm64
diff --git a/debian/tests/stretch b/debian/tests/stretch
index cbb25ca..3ec6f97 100755
--- a/debian/tests/stretch
+++ b/debian/tests/stretch
@@ -6,13 +6,22 @@ timestamp='2017-01-01T00:00:00Z'
expectedEpoch='1483228800'
+# https://people.debian.org/~tianon/debuerreotype/
declare -A expectedSha256s=(
- ['amd64']='26490ed3400a5029b8b5939c6cebd38691e28ea5c616bb54f25f758125417c5f'
- ['arm64']='45e5e0c6da27db14de19a600663140db1f85f7516a856bf00957436e93e1f684'
+ ['amd64']='cff511450d3ddec4defc51be1850ca44e527e126aabfa1b6649354f2f2a568f4'
+ ['arm64']='81ad4f5e7c10b15fc8389c1386e1fe1b2d05bc61d404e10473f5f5a0acde464c'
+ ['armel']='596117e125f6e0828471564bd5d4f9bdd1ed654f7d84c21e7ab9d640e789763f'
+ ['armhf']='9e0f63a3d5442fcd1408d61a9c592355138334ddc5c3fb471d5f131511cf286c'
+ ['i386']='b4e574904703114291f1c42bd80744b1da333d95cf5c00e8e71ba09241ca66ae'
+ ['mips64el']='767982e9ca353058d9ca44f410e0367071ebf9938a4a996e3ed9139d57bc987f'
+ ['ppc64el']='99bc8a072448360be9ff741b98466835bc0f270041784cacbcb3803f8d3fbb6a'
+ ['s390x']='7ad6fa1abaf9d8cb3fd9d1257e24363bc842f0fe204e98d0a2aa995541b62b1c'
)
dpkgArch="$(dpkg --print-architecture)"
expectedSha256="${expectedSha256s["$dpkgArch"]:-}"
+expectedCompareUrl="https://people.debian.org/~tianon/debuerreotype/$suite--$timestamp--0.14--$dpkgArch--$expectedSha256.txz"
+
tempDir="$(mktemp -d)"
trap "rm -rf '$tempDir'" EXIT
rootfs="$tempDir/rootfs"
@@ -55,11 +64,11 @@ if [ "$sha256" != "$expectedSha256" ]; then
set +x
echo >&2
echo >&2 'ERROR: expected SHA256 does not match actual -- downloading pristine source to compare (via diffoscope)'
+ echo >&2 " - $expectedCompareUrl"
echo >&2
)
- toCompare="https://people.debian.org/~tianon/debuerreotype/$suite--$timestamp--$dpkgArch--$expectedSha256.txz"
- wget -qO "$tempDir/expected.txz" "$toCompare"
+ wget -qO "$tempDir/expected.txz" "$expectedCompareUrl"
xz -d < "$tempDir/expected.txz" > "$tempDir/expected.tar"
diffoscope >&2 "$tempDir/expected.tar" "$tempDir/actual.tar"
exit 1
diff --git a/docker-run.sh b/docker-run.sh
new file mode 100755
index 0000000..f5db052
--- /dev/null
+++ b/docker-run.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# usage: mkdir -p output && ./run-script.sh ./examples/debian.sh output ...
+
+thisDir="$(readlink -vf "$BASH_SOURCE")"
+thisDir="$(dirname "$thisDir")"
+
+source "$thisDir/scripts/.constants.sh" \
+ --flags 'image:' \
+ --flags 'no-bind' \
+ --flags 'no-build' \
+ --flags 'pull' \
+ -- \
+ '[--image=foo/bar:baz] [--no-build] [--no-bind] [--pull] [script/command]' \
+ './examples/debian.sh output stretch 2017-05-08T00:00:00Z
+--no-build --image=debuerreotype:ubuntu ./examples/ubuntu.sh output xenial'
+
+eval "$dgetopt"
+image=
+build=1
+bindMount=1
+pull=
+while true; do
+ flag="$1"; shift
+ dgetopt-case "$flag"
+ case "$flag" in
+ --image) image="$1"; shift ;;
+ --no-bind) bindMount= ;;
+ --no-build) build= ;;
+ --pull) pull=1 ;;
+ --) break ;;
+ *) eusage "unknown flag '$flag'" ;;
+ esac
+done
+
+if [ -z "$image" ]; then
+ image="$("$thisDir/.docker-image.sh")"
+fi
+if [ -n "$build" ]; then
+ docker build ${pull:+--pull} --tag "$image" "$thisDir"
+elif [ -n "$pull" ]; then
+ docker pull "$image"
+else
+ # make sure "docker run" doesn't pull (we have `--no-build` and no explicit `--pull`)
+ docker image inspect "$image" > /dev/null
+fi
+
+args=(
+ --hostname debuerreotype
+ --init
+ --interactive
+ --rm
+
+ # we ought to be able to mount/unshare
+ --cap-add SYS_ADMIN
+ # make sure we don't get extended attributes
+ --cap-drop SETFCAP
+
+ # AppArmor also blocks mount/unshare :)
+ --security-opt apparmor=unconfined
+
+ # --debian-eol potato wants to run "chroot ... mount ... /proc" which gets blocked (i386, ancient binaries, blah blah blah)
+ --security-opt seccomp=unconfined
+ # (other arches see this occasionally too)
+
+ --tmpfs /tmp:dev,exec,suid,noatime
+ --env TMPDIR=/tmp
+
+ --workdir /workdir
+)
+if [ -n "$bindMount" ]; then
+ args+=( --mount "type=bind,src=$PWD,dst=/workdir" )
+else
+ args+=( --volume /workdir )
+fi
+
+if [ -t 0 ] && [ -t 1 ]; then
+ args+=( --tty )
+fi
+
+exec docker run "${args[@]}" "$image" "$@"
diff --git a/examples/debian-all.sh b/examples/debian-all.sh
new file mode 100755
index 0000000..e39472c
--- /dev/null
+++ b/examples/debian-all.sh
@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+suites=(
+ unstable
+ testing
+ stable
+ oldstable
+ oldoldstable
+
+ # just in case (will no-op with "not supported on 'arch'" unless it exists)
+ oldoldoldstable
+)
+
+debuerreotypeScriptsDir="$(which debuerreotype-init)"
+debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")"
+debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")"
+
+source "$debuerreotypeScriptsDir/.constants.sh" \
+ --flags 'arch:' \
+ -- \
+ '[--arch=<arch>] <output-dir> <timestamp>' \
+ 'output 2017-05-08T00:00:00Z'
+
+eval "$dgetopt"
+arch=
+while true; do
+ flag="$1"; shift
+ dgetopt-case "$flag"
+ case "$flag" in
+ --arch) arch="$1"; shift ;;
+ --) break ;;
+ *) eusage "unknown flag '$flag'" ;;
+ esac
+done
+
+outputDir="${1:-}"; shift || eusage 'missing output-dir'
+timestamp="${1:-}"; shift || eusage 'missing timestamp'
+
+debianArgs=( --codename-copy )
+
+mirror="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "$timestamp")"
+secmirror="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "$timestamp" 'debian-security')"
+
+dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}"
+echo
+echo "-- BUILDING TARBALLS FOR '$dpkgArch' FROM '$mirror/' --"
+echo
+debianArgs+=( --arch="$dpkgArch" )
+
+thisDir="$(readlink -vf "$BASH_SOURCE")"
+thisDir="$(dirname "$thisDir")"
+
+_eol-date() {
+ local codename="$1"; shift # "bullseye", "buster", etc.
+ if [ ! -s /usr/share/distro-info/debian.csv ]; then
+ echo >&2 "warning: looks like we are missing 'distro-info-data' (/usr/share/distro-info/debian.csv); cannot calculate EOL dates accurately!"
+ exit 1
+ fi
+ awk -F, -v codename="$codename" '
+ NR == 1 {
+ headers = NF
+ for (i = 1; i <= headers; i++) {
+ header[i] = $i
+ }
+ next
+ }
+ {
+ delete row
+ for (i = 1; i <= NF && i <= headers; i++) {
+ row[header[i]] = $i
+ }
+ }
+ row["series"] == codename {
+ if (row["eol-lts"] != "") {
+ eol = row["eol-lts"]
+ exit 0
+ }
+ if (row["eol"] != "") {
+ eol = row["eol"]
+ exit 0
+ }
+ exit 1
+ }
+ END {
+ if (eol != "") {
+ print eol
+ exit 0
+ }
+ exit 1
+ }
+ ' /usr/share/distro-info/debian.csv
+}
+
+_codename() {
+ local dist="$1"; shift
+
+ local release
+ if release="$(wget --quiet --output-document=- "$mirror/dists/$dist/InRelease")"; then
+ :
+ elif release="$(wget --quiet --output-document=- "$mirror/dists/$dist/Release")"; then
+ :
+ else
+ return 1
+ fi
+
+ local codename
+ codename="$(awk '$1 == "Codename:" { print $2 }' <<<"$release")"
+ [ -n "$codename" ] || return 1
+ echo "$codename"
+}
+
+_check() {
+ local host="$1"; shift # "$mirror", "$secmirror"
+ local dist="$1"; shift # "$suite-security", "$suite/updates", "$suite"
+ local comp="${1:-main}"
+
+ if wget --quiet --spider "$host/dists/$dist/$comp/binary-$dpkgArch/Packages.xz"; then
+ return 0
+ fi
+
+ if wget --quiet --spider "$host/dists/$dist/$comp/binary-$dpkgArch/Packages.gz"; then
+ return 0
+ fi
+
+ return 1
+}
+
+for suite in "${suites[@]}"; do
+ doSkip=
+ case "$suite" in
+ testing | unstable) ;;
+
+ *)
+ # https://lists.debian.org/debian-devel-announce/2019/07/msg00004.html
+ if \
+ ! _check "$secmirror" "$suite-security" \
+ && ! _check "$secmirror" "$suite/updates" \
+ ; then
+ doSkip=1
+ fi
+ if [ -z "$doSkip" ] && codename="$(_codename "$suite")" && eol="$(_eol-date "$codename")"; then
+ epoch="$(date --date "$timestamp" '+%s')"
+ eolEpoch="$(date --date "$eol" '+%s')"
+ if [ "$epoch" -ge "$eolEpoch" ]; then
+ echo >&2
+ echo >&2 "warning: '$suite' ('$codename') is EOL at '$timestamp' ('$eol'); skipping"
+ echo >&2
+ continue
+ fi
+ fi
+ ;;
+ esac
+ if ! _check "$mirror" "$suite"; then
+ doSkip=1
+ fi
+ if [ -n "$doSkip" ]; then
+ echo >&2
+ echo >&2 "warning: '$suite' not supported on '$dpkgArch' (at '$timestamp'); skipping"
+ echo >&2
+ continue
+ fi
+ "$thisDir/debian.sh" "${debianArgs[@]}" "$outputDir" "$suite" "$timestamp"
+done
diff --git a/examples/debian.sh b/examples/debian.sh
new file mode 100755
index 0000000..c679609
--- /dev/null
+++ b/examples/debian.sh
@@ -0,0 +1,363 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+debuerreotypeScriptsDir="$(which debuerreotype-init)"
+debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")"
+debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")"
+
+source "$debuerreotypeScriptsDir/.constants.sh" \
+ --flags 'codename-copy' \
+ --flags 'eol,ports' \
+ --flags 'arch:' \
+ --flags 'include:,exclude:' \
+ -- \
+ '[--codename-copy] [--eol] [--ports] [--arch=<arch>] <output-dir> <suite> <timestamp>' \
+ 'output stretch 2017-05-08T00:00:00Z
+--codename-copy output stable 2017-05-08T00:00:00Z
+--eol output squeeze 2016-03-14T00:00:00Z
+--eol --arch i386 output sarge 2016-03-14T00:00:00Z'
+
+eval "$dgetopt"
+codenameCopy=
+eol=
+ports=
+include=
+exclude=
+arch=
+while true; do
+ flag="$1"; shift
+ dgetopt-case "$flag"
+ case "$flag" in
+ --codename-copy) codenameCopy=1 ;; # for copying a "stable.tar.xz" to "stretch.tar.xz" with updated sources.list (saves a lot of extra building work)
+ --eol) eol=1 ;; # for using "archive.debian.org"
+ --ports) ports=1 ;; # for using "debian-ports"
+ --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init
+ --include) include="${include:+$include,}$1"; shift ;;
+ --exclude) exclude="${exclude:+$exclude,}$1"; shift ;;
+ --) break ;;
+ *) eusage "unknown flag '$flag'" ;;
+ esac
+done
+
+outputDir="${1:-}"; shift || eusage 'missing output-dir'
+suite="${1:-}"; shift || eusage 'missing suite'
+timestamp="${1:-}"; shift || eusage 'missing timestamp'
+
+set -x
+
+outputDir="$(readlink -ve "$outputDir")"
+
+tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")"
+trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT
+
+export TZ='UTC' LC_ALL='C'
+
+epoch="$(date --date "$timestamp" +%s)"
+serial="$(date --date "@$epoch" +%Y%m%d)"
+dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}"
+
+exportDir="$tmpDir/output"
+archDir="$exportDir/$serial/$dpkgArch"
+tmpOutputDir="$archDir/$suite"
+
+touch_epoch() {
+ while [ "$#" -gt 0 ]; do
+ local f="$1"; shift
+ touch --no-dereference --date="@$epoch" "$f"
+ done
+}
+
+for archive in '' security; do
+ snapshotUrlFile="$archDir/snapshot-url${archive:+-${archive}}"
+ mirrorArgs=()
+ if [ -n "$ports" ]; then
+ mirrorArgs+=( --ports )
+ fi
+ if [ -n "$eol" ]; then
+ mirrorArgs+=( --eol )
+ fi
+ mirrorArgs+=( "@$epoch" "$suite${archive:+-$archive}" "$dpkgArch" main )
+ if ! mirrors="$("$debuerreotypeScriptsDir/.debian-mirror.sh" "${mirrorArgs[@]}")"; then
+ if [ "$archive" = 'security' ]; then
+ # if we fail to find the security mirror, we're probably not security supported (which is ~fine)
+ continue
+ else
+ exit 1
+ fi
+ fi
+ eval "$mirrors"
+ [ -n "$snapshotMirror" ]
+ snapshotUrlDir="$(dirname "$snapshotUrlFile")"
+ mkdir -p "$snapshotUrlDir"
+ echo "$snapshotMirror" > "$snapshotUrlFile"
+ touch_epoch "$snapshotUrlFile"
+done
+
+initArgs=(
+ --arch "$dpkgArch"
+)
+
+if [ -z "$eol" ]; then
+ initArgs+=( --debian )
+else
+ initArgs+=( --debian-eol )
+fi
+if [ -n "$ports" ]; then
+ initArgs+=(
+ --debian-ports
+ --include=debian-ports-archive-keyring
+ )
+fi
+
+export GNUPGHOME="$tmpDir/gnupg"
+mkdir -p "$GNUPGHOME"
+keyring="$tmpDir/debian-archive-$suite-keyring.gpg"
+if [ "$suite" = 'slink' ]; then
+ # slink (2.1) introduced apt, but without PGP 😅
+ initArgs+=( --no-check-gpg )
+elif [ "$suite" = 'potato' ]; then
+ # src:debian-archive-keyring was created in 2006, thus does not include a key for potato (2.2; EOL in 2003)
+ gpg --batch --no-default-keyring --keyring "$keyring" \
+ --keyserver keyserver.ubuntu.com \
+ --recv-keys 8FD47FF1AA9372C37043DC28AA7DEB7B722F1AED
+ initArgs+=( --keyring "$keyring" )
+else
+ # check against all releases (ie, combine both "debian-archive-keyring.gpg" and "debian-archive-removed-keys.gpg"), since we cannot really know whether the target release became EOL later than the snapshot date we are targeting
+ gpg --batch --no-default-keyring --keyring "$keyring" --import \
+ /usr/share/keyrings/debian-archive-keyring.gpg \
+ /usr/share/keyrings/debian-archive-removed-keys.gpg
+ if [ -n "$ports" ]; then
+ gpg --batch --no-default-keyring --keyring "$keyring" --import \
+ /usr/share/keyrings/debian-ports-archive-keyring.gpg \
+ /usr/share/keyrings/debian-ports-archive-keyring-removed.gpg
+ fi
+ initArgs+=( --keyring "$keyring" )
+fi
+
+mkdir -p "$tmpOutputDir"
+
+mirror="$(< "$archDir/snapshot-url")"
+if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then
+ gpgv \
+ --keyring "$keyring" \
+ --output "$tmpOutputDir/Release" \
+ "$tmpOutputDir/InRelease"
+ [ -s "$tmpOutputDir/Release" ]
+elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then
+ rm -f "$tmpOutputDir/InRelease" # remove wget leftovers
+ gpgv \
+ --keyring "$keyring" \
+ "$tmpOutputDir/Release.gpg" \
+ "$tmpOutputDir/Release"
+ [ -s "$tmpOutputDir/Release" ]
+elif [ "$suite" = 'slink' ]; then
+ # "Release" files were introduced in potato (2.2+)
+ rm -f "$tmpOutputDir/InRelease" "$tmpOutputDir/Release.gpg" "$tmpOutputDir/Release" # remove wget leftovers
+else
+ echo >&2 "error: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')"
+ exit 1
+fi
+codename=
+if [ -f "$tmpOutputDir/Release" ]; then
+ codename="$(awk -F ': ' '$1 == "Codename" { print $2; exit }' "$tmpOutputDir/Release")"
+fi
+if [ -n "$codenameCopy" ] && [ "$codename" = "$suite" ]; then
+ # if codename already is the same as suite, then making a copy does not make any sense
+ codenameCopy=
+fi
+if [ -n "$codenameCopy" ] && [ -z "$codename" ]; then
+ echo >&2 "error: --codename-copy specified but we failed to get a Codename for $suite"
+ exit 1
+fi
+
+initArgs+=(
+ # disable merged-usr (for now?) due to the following compelling arguments:
+ # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc)
+ # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still)
+ # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406
+ --no-merged-usr
+)
+
+if [ -n "$include" ]; then
+ initArgs+=( --include="$include" )
+fi
+if [ -n "$exclude" ]; then
+ initArgs+=( --exclude="$exclude" )
+fi
+
+rootfsDir="$tmpDir/rootfs"
+debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "@$epoch"
+
+if [ -n "$eol" ]; then
+ debuerreotype-gpgv-ignore-expiration-config "$rootfsDir"
+fi
+
+debuerreotype-minimizing-config "$rootfsDir"
+
+debuerreotype-apt-get "$rootfsDir" update -qq
+
+aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")"
+if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then
+ debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq
+else
+ debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq
+fi
+
+if dpkg --compare-versions "$aptVersion" '>=' '0.7.14~'; then
+ # https://salsa.debian.org/apt-team/apt/commit/06d79436542ccf3e9664306da05ba4c34fba4882
+ noInstallRecommends='--no-install-recommends'
+else
+ # etch (4.0) and lower do not support --no-install-recommends
+ noInstallRecommends='-o APT::Install-Recommends=0'
+fi
+
+if [ -n "$eol" ] && dpkg --compare-versions "$aptVersion" '>=' '0.7.26~'; then
+ # https://salsa.debian.org/apt-team/apt/commit/1ddb859611d2e0f3d9ea12085001810f689e8c99
+ echo 'Acquire::Check-Valid-Until "false";' > "$rootfsDir"/etc/apt/apt.conf.d/check-valid-until.conf
+ # TODO make this a real script so it can have a nice comment explaining why we do it for EOL releases?
+fi
+
+# copy the rootfs to create other variants
+mkdir "$rootfsDir"-slim
+tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim
+
+# for historical reasons (related to their usefulness in debugging non-working container networking in container early days before "--network container:xxx"), Debian 10 and older non-slim images included both "ping" and "ip" above "minbase", but in 11+ (Bullseye), that will no longer be the case and we will instead be a faithful minbase again :D
+epoch2021="$(date --date '2021-01-01 00:00:00' +%s)"
+if [ "$epoch" -lt "$epoch2021" ] || { isDebianBusterOrOlder="$([ -f "$rootfsDir/etc/os-release" ] && source "$rootfsDir/etc/os-release" && [ -n "${VERSION_ID:-}" ] && [ "${VERSION_ID%%.*}" -le 10 ] && echo 1)" && [ -n "$isDebianBusterOrOlder" ]; }; then
+ # prefer iproute2 if it exists
+ iproute=iproute2
+ if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then
+ # poor wheezy
+ iproute=iproute
+ fi
+ ping=iputils-ping
+ if debuerreotype-chroot "$rootfsDir" bash -c 'command -v ping > /dev/null'; then
+ # if we already have "ping" (as in potato, 2.2), skip installing any extra ping package
+ ping=
+ fi
+ debuerreotype-apt-get "$rootfsDir" install -y $noInstallRecommends $ping $iproute
+fi
+
+debuerreotype-slimify "$rootfsDir"-slim
+
+sourcesListArgs=()
+[ -z "$eol" ] || sourcesListArgs+=( --eol )
+[ -z "$ports" ] || sourcesListArgs+=( --ports )
+
+create_artifacts() {
+ local targetBase="$1"; shift
+ local rootfs="$1"; shift
+ local suite="$1"; shift
+ local variant="$1"; shift
+
+ # make a copy of the snapshot-facing sources.list file before we overwrite it
+ cp "$rootfs/etc/apt/sources.list" "$targetBase.sources-list-snapshot"
+ touch_epoch "$targetBase.sources-list-snapshot"
+
+ debuerreotype-debian-sources-list "${sourcesListArgs[@]}" "$rootfs" "$suite"
+
+ local tarArgs=(
+ # https://www.freedesktop.org/software/systemd/man/machine-id.html
+ --exclude ./etc/machine-id
+ # "debuerreotype-fixup" will make this an empty file for reproducibility, but for our Docker images it seems more appropriate for it to not exist (since they've never actually been "booted" so having the "first boot" logic trigger if someone were to run systemd in them conceptually makes sense)
+ )
+
+ case "$suite" in
+ sarge) # 3.1
+ # for some reason, sarge creates "/var/cache/man/index.db" with some obvious embedded unix timestamps (but if we exclude it, "man" still works properly, so *shrug*)
+ tarArgs+=( --exclude ./var/cache/man/index.db )
+ ;;
+
+ woody) # 3.0
+ # woody not only contains "exim", but launches it during our build process and tries to email "root at debuerreotype" (which fails and creates non-reproducibility)
+ tarArgs+=( --exclude ./var/spool/exim --exclude ./var/log/exim )
+ ;;
+
+ potato) # 2.2
+ tarArgs+=(
+ # for some reason, pototo leaves a core dump (TODO figure out why??)
+ --exclude './core'
+ # also, it leaves some junk in /tmp (/tmp/fdmount.conf.tmp.XXX)
+ --exclude './tmp/fdmount.conf.tmp.*'
+ )
+ ;;
+
+ slink) # 2.1
+ tarArgs+=(
+ # same as potato :(
+ --exclude './tmp/fdmount.conf.tmp.*'
+ )
+ ;;
+ esac
+
+ debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz"
+ du -hsx "$targetBase.tar.xz"
+
+ sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256"
+ touch_epoch "$targetBase.tar.xz.sha256"
+
+ debuerreotype-chroot "$rootfs" bash -c '
+ if ! dpkg-query -W 2> /dev/null; then
+ # --debian-eol woody has no dpkg-query
+ dpkg -l
+ fi
+ ' > "$targetBase.manifest"
+ echo "$suite" > "$targetBase.apt-dist"
+ echo "$dpkgArch" > "$targetBase.dpkg-arch"
+ echo "$epoch" > "$targetBase.debuerreotype-epoch"
+ echo "$variant" > "$targetBase.debuerreotype-variant"
+ debuerreotype-version > "$targetBase.debuerreotype-version"
+ touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*}
+
+ for f in debian_version os-release apt/sources.list; do
+ targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")"
+ if [ -e "$rootfs/etc/$f" ]; then
+ # /etc/os-release does not exist in --debian-eol squeeze, for example (hence the existence check)
+ cp "$rootfs/etc/$f" "$targetFile"
+ touch_epoch "$targetFile"
+ fi
+ done
+}
+
+for rootfs in "$rootfsDir"*/; do
+ rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ...
+
+ du -hsx "$rootfs"
+
+ variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ...
+ variant="${variant#rootfs}" # "", "-slim", ...
+ variant="${variant#-}" # "", "slim", ...
+
+ variantDir="$tmpOutputDir/$variant"
+ mkdir -p "$variantDir"
+
+ targetBase="$variantDir/rootfs"
+
+ create_artifacts "$targetBase" "$rootfs" "$suite" "$variant"
+done
+
+if [ -n "$codenameCopy" ]; then
+ codenameDir="$archDir/$codename"
+ mkdir -p "$codenameDir"
+ tar -cC "$tmpOutputDir" --exclude='**/rootfs.*' . | tar -xC "$codenameDir"
+
+ for rootfs in "$rootfsDir"*/; do
+ rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ...
+
+ variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ...
+ variant="${variant#rootfs}" # "", "-slim", ...
+ variant="${variant#-}" # "", "slim", ...
+
+ variantDir="$codenameDir/$variant"
+ targetBase="$variantDir/rootfs"
+
+ # point sources.list back at snapshot.debian.org temporarily (but this time pointing at $codename instead of $suite)
+ debuerreotype-debian-sources-list --snapshot "${sourcesListArgs[@]}" "$rootfs" "$codename"
+
+ create_artifacts "$targetBase" "$rootfs" "$codename" "$variant"
+ done
+fi
+
+user="$(stat --format '%u' "$outputDir")"
+group="$(stat --format '%g' "$outputDir")"
+tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir"
diff --git a/examples/oci-image.sh b/examples/oci-image.sh
new file mode 100755
index 0000000..df8f705
--- /dev/null
+++ b/examples/oci-image.sh
@@ -0,0 +1,279 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# create https://github.com/opencontainers/image-spec/blob/v1.0.1/image-layout.md (+ Docker's "manifest.json") from the output of "debian.sh"
+# the resulting file is suitable for "ctr image import" or "docker load"
+
+# (this can *technically* run via "docker-run.sh", but IMO it's much easier to run unprivileged on the host)
+# RUN apt-get update \
+# && apt-get install -y jq pigz \
+# && rm -rf /var/lib/apt/lists/*
+
+thisDir="$(readlink -vf "$BASH_SOURCE")"
+thisDir="$(dirname "$thisDir")"
+
+if [ -x "$thisDir/../scripts/debuerreotype-init" ]; then
+ debuerreotypeScriptsDir="$(dirname "$thisDir")/scripts"
+else
+ debuerreotypeScriptsDir="$(which debuerreotype-init)"
+ debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")"
+ debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")"
+fi
+
+source "$debuerreotypeScriptsDir/.constants.sh" \
+ --flags 'meta:' \
+ -- \
+ '<target-file.tar> <source-directory>' \
+ 'out/oci-unstable-slim.tar out/20210511/amd64/unstable/slim'
+
+eval "$dgetopt"
+meta=
+while true; do
+ flag="$1"; shift
+ dgetopt-case "$flag"
+ case "$flag" in
+ --meta) meta="$1"; shift ;;
+ --) break ;;
+ *) eusage "unknown flag '$flag'" ;;
+ esac
+done
+
+targetFile="${1:-}"; shift || eusage 'missing target-file' # "something.tar"
+sourceDir="${1:-}"; shift || eusage 'missing source-directory' # "out/YYYYMMDD/ARCH/SUITE{,/slim}"
+
+targetFile="$(readlink -vf "$targetFile")"
+if [ -n "$meta" ]; then
+ meta="$(readlink -vf "$meta")"
+fi
+sourceDir="$(readlink -ve "$sourceDir")"
+
+tempDir="$(mktemp -d)"
+trap "$(printf 'rm -rf %q' "$tempDir")" EXIT
+
+mkdir -p "$tempDir/oci/blobs/sha256"
+jq -ncS '{ imageLayoutVersion: "1.0.0" }' > "$tempDir/oci/oci-layout"
+
+version="$(< "$sourceDir/rootfs.debuerreotype-version")"
+epoch="$(< "$sourceDir/rootfs.debuerreotype-epoch")"
+iso8601="$(date --date="@$epoch" '+%Y-%m-%dT%H:%M:%SZ')"
+export version epoch iso8601
+
+if [ -s "$sourceDir/rootfs.apt-dist" ]; then
+ suite="$(< "$sourceDir/rootfs.apt-dist")"
+ # TODO remove this fallback once debuerreotype 0.13 is released and we can safely assume "rootfs.apt-dist" exists
+else
+ suite="$(awk '$1 == "deb" { print $3; exit }' "$sourceDir/rootfs.sources-list")"
+fi
+export suite
+
+if [ -s "$sourceDir/rootfs.debuerreotype-variant" ]; then
+ variant="$(< "$sourceDir/rootfs.debuerreotype-variant")"
+ # TODO remove this fallback once debuerreotype 0.13 is released and we can safely assume "rootfs.debuerreotype-variant" exists
+else
+ dirBase="$(basename "$sourceDir")"
+ case "$dirBase" in
+ slim) variant="$dirBase" ;;
+ "$suite") variant='' ;;
+ *) echo >&2 "error: unknown variant: '$variant'"; exit 1 ;;
+ esac
+fi
+
+if [ -s "$sourceDir/rootfs.dpkg-arch" ]; then
+ dpkgArch="$(< "$sourceDir/rootfs.dpkg-arch")"
+ # TODO remove these fallbacks once debuerreotype 0.13 is released and we can safely assume "rootfs.dpkg-arch" exists
+elif [ -n "$variant" ]; then # xxx/YYYYMMDD/ARCH/SUITE/slim
+ dpkgArch="$(cd "$sourceDir/../.." && basename "$PWD")"
+else # xxx/YYYYMMDD/ARCH/SUITE
+ dpkgArch="$(cd "$sourceDir/.." && basename "$PWD")"
+fi
+unset goArch
+goArm=
+case "$dpkgArch" in
+ amd64 | arm64 | s390x | riscv64) goArch="$dpkgArch" ;;
+ armel | arm) goArch='arm'; goArm='5' ;;
+ armhf) goArch='arm'; if grep -qi raspbian "$sourceDir/rootfs.os-release"; then goArm='6'; else goArm='7'; fi ;;
+ i386) goArch='386' ;;
+ mips64el | ppc64el) goArch="${dpkgArch%el}le" ;;
+ *) echo >&2 "error: unknown dpkg architecture: '$dpkgArch'"; exit 1 ;;
+esac
+unset bashbrewArch
+case "$goArch" in
+ 386) bashbrewArch='i386' ;;
+ amd64 | mips64le | ppc64le | riscv64 | s390x) bashbrewArch="$goArch" ;;
+ arm) bashbrewArch="${goArch}32v${goArm}" ;;
+ arm64) bashbrewArch="${goArch}v8" ;;
+ *) echo >&2 "error: unknown Go architecture: '$goArch'"; exit 1 ;;
+esac
+export dpkgArch goArch goArm bashbrewArch
+
+osID="$(id="$(grep -E '^ID=' "$sourceDir/rootfs.os-release")" && eval "$id" && echo "${ID:-}")" || : # "debian", "raspbian", "ubuntu", etc
+: "${osID:=debian}" # if for some reason the above fails, fall back to "debian"
+
+echo >&2 "processing $osID '$suite'${variant:+", variant '$variant'"}, architecture '$dpkgArch' ('$bashbrewArch')"
+
+_sha256() {
+ sha256sum "$@" | cut -d' ' -f1
+}
+
+echo >&2 "decompressing rootfs (xz) ..."
+
+xz --decompress --threads=0 --stdout "$sourceDir/rootfs.tar.xz" > "$tempDir/rootfs.tar"
+diffId="$(_sha256 "$tempDir/rootfs.tar")"
+export diffId="sha256:$diffId"
+
+echo >&2 "recompressing rootfs (gzip) ..."
+
+pigz --best --no-time "$tempDir/rootfs.tar"
+rootfsSize="$(stat --format='%s' "$tempDir/rootfs.tar.gz")"
+rootfsSha256="$(_sha256 "$tempDir/rootfs.tar.gz")"
+export rootfsSize rootfsSha256
+mv "$tempDir/rootfs.tar.gz" "$tempDir/oci/blobs/sha256/$rootfsSha256"
+
+script='debian.sh'
+if [ -x "$thisDir/$osID.sh" ]; then
+ script="$osID.sh"
+fi
+export script
+
+echo >&2 "generating config ..."
+
+# https://github.com/opencontainers/image-spec/blob/v1.0.1/config.md
+jq -ncS '
+ {
+ config: {
+ Env: [ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ],
+ Entrypoint: [],
+ Cmd: [ "bash" ],
+ },
+ created: env.iso8601,
+ history: [
+ {
+ created: env.iso8601,
+ created_by: (
+ "# " + env.script
+ + if env.script != "raspbian.sh" then
+ " --arch " + (env.dpkgArch | @sh)
+ else "" end
+ + " out/ "
+ + (env.suite | @sh)
+ + if env.script == "debian.sh" then
+ " "
+ else
+ " # "
+ end
+ + ("@" + env.epoch | @sh)
+ ),
+ comment: ( "debuerreotype " + env.version ),
+ }
+ ],
+ rootfs: {
+ type: "layers",
+ diff_ids: [ env.diffId ],
+ },
+ os: "linux",
+ architecture: env.goArch,
+ }
+ + if env.goArch == "arm64" then
+ { variant: "v8" }
+ elif env.goArch == "arm" then
+ { variant: ( "v" + env.goArm ) }
+ else
+ {}
+ end
+' > "$tempDir/config.json"
+configSize="$(stat --format='%s' "$tempDir/config.json")"
+configSha256="$(_sha256 "$tempDir/config.json")"
+export configSize configSha256
+mv "$tempDir/config.json" "$tempDir/oci/blobs/sha256/$configSha256"
+
+# https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md
+jq -ncS '
+ {
+ schemaVersion: 2,
+ mediaType: "application/vnd.docker.distribution.manifest.v2+json",
+ config: {
+ mediaType: "application/vnd.docker.container.image.v1+json",
+ size: (env.configSize | tonumber),
+ digest: ( "sha256:" + env.configSha256 ),
+ },
+ layers: [
+ {
+ mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
+ size: (env.rootfsSize | tonumber),
+ digest: ( "sha256:" + env.rootfsSha256 ),
+ }
+ ],
+ }
+' > "$tempDir/manifest.json"
+manifestSize="$(stat --format='%s' "$tempDir/manifest.json")"
+manifestSha256="$(_sha256 "$tempDir/manifest.json")"
+export manifestSize manifestSha256
+mv "$tempDir/manifest.json" "$tempDir/oci/blobs/sha256/$manifestSha256"
+
+export repo="$bashbrewArch/$osID" # "amd64/debian", "arm32v6/raspbian", etc.
+export tag="$suite${variant:+-$variant}" # "buster", "buster-slim", etc.
+export image="$repo:$tag"
+
+# https://github.com/opencontainers/image-spec/blob/v1.0.1/image-index.md
+jq -ncS '
+ {
+ schemaVersion: 2,
+ manifests: [
+ {
+ mediaType: "application/vnd.docker.distribution.manifest.v2+json",
+ size: (env.manifestSize | tonumber),
+ digest: ( "sha256:" + env.manifestSha256 ),
+ annotations: {
+ "io.containerd.image.name": env.image,
+ "org.opencontainers.image.ref.name": env.tag,
+ },
+ }
+ ],
+ }
+' > "$tempDir/oci/index.json"
+
+# Docker's "manifest.json" so that we can "docker load" the result of this script too
+jq -ncS '
+ [
+ {
+ Config: ( "blobs/sha256/" + env.configSha256 ),
+ Layers: [ "blobs/sha256/" + env.rootfsSha256 ],
+ RepoTags: [ env.image ],
+ }
+ ]
+' > "$tempDir/oci/manifest.json"
+
+echo >&2 "fixing timestamps ..."
+
+find "$tempDir/oci" \
+ -newermt "@$epoch" \
+ -exec touch --no-dereference --date="@$epoch" '{}' +
+
+echo >&2 "generating tarball ($targetFile) ..."
+
+tar --create \
+ --auto-compress \
+ --directory "$tempDir/oci" \
+ --file "$targetFile" \
+ --numeric-owner --owner 1000:1000 \
+ --sort name \
+ .
+touch --no-dereference --date="@$epoch" "$targetFile"
+
+platform="$(jq -c 'with_entries(select(.key == ([ "os", "architecture", "variant" ][])))' "$tempDir/oci/blobs/sha256/$configSha256")"
+jq -n --argjson platform "$platform" '
+ {
+ image: env.image,
+ repo: env.repo,
+ tag: env.tag,
+ id: ( "sha256:" + env.configSha256 ),
+ digest: ( "sha256:" + env.manifestSha256 ),
+ platform: $platform,
+ }
+' > "$tempDir/meta.json"
+touch --no-dereference --date="@$epoch" "$tempDir/meta.json"
+if [ -n "$meta" ]; then
+ cp -a "$tempDir/meta.json" "$meta"
+fi
+
+jq >&2 . "$tempDir/meta.json"
diff --git a/examples/raspbian.sh b/examples/raspbian.sh
new file mode 100755
index 0000000..38916be
--- /dev/null
+++ b/examples/raspbian.sh
@@ -0,0 +1,198 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# # https://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/
+# RUN wget -O raspbian.deb 'https://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring-udeb_20120528.2_all.udeb' \
+# && apt-get install -y ./raspbian.deb \
+# && rm raspbian.deb
+
+debuerreotypeScriptsDir="$(which debuerreotype-init)"
+debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")"
+debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")"
+
+source "$debuerreotypeScriptsDir/.constants.sh" \
+ -- \
+ '<output-dir> <suite>' \
+ 'output stretch'
+
+eval "$dgetopt"
+while true; do
+ flag="$1"; shift
+ dgetopt-case "$flag"
+ case "$flag" in
+ --) break ;;
+ *) eusage "unknown flag '$flag'" ;;
+ esac
+done
+
+outputDir="${1:-}"; shift || eusage 'missing output-dir'
+suite="${1:-}"; shift || eusage 'missing suite'
+
+set -x
+
+outputDir="$(readlink -ve "$outputDir")"
+
+tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")"
+trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT
+
+export TZ='UTC' LC_ALL='C'
+
+dpkgArch='armhf'
+
+exportDir="$tmpDir/output"
+archDir="$exportDir/raspbian/$dpkgArch"
+tmpOutputDir="$archDir/$suite"
+
+#mirror='http://archive.raspbian.org/raspbian'
+mirror='http://mirrordirector.raspbian.org/raspbian'
+# (https://www.raspbian.org/RaspbianMirrors#The_mirror_redirection_system)
+
+initArgs=(
+ --arch "$dpkgArch"
+ --non-debian
+)
+
+export GNUPGHOME="$tmpDir/gnupg"
+mkdir -p "$GNUPGHOME"
+keyring='/usr/share/keyrings/raspbian-archive-keyring.gpg'
+if [ ! -s "$keyring" ]; then
+ # since we're using mirrors, we ought to be more explicit about download verification
+ keyUrl='https://archive.raspbian.org/raspbian.public.key'
+ (
+ set +x
+ echo >&2
+ echo >&2 "WARNING: missing '$keyring' (from 'raspbian-archive-keyring' package)"
+ echo >&2 " downloading '$keyUrl' (without verification beyond TLS)!"
+ echo >&2
+ )
+ sleep 5
+ keyring="$tmpDir/raspbian-archive-keyring.gpg"
+ wget -O "$keyring.asc" "$keyUrl"
+ gpg --batch --no-default-keyring --keyring "$keyring" --import "$keyring.asc"
+ rm -f "$keyring.asc"
+fi
+initArgs+=( --keyring "$keyring" )
+
+mkdir -p "$tmpOutputDir"
+
+if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then
+ gpgv \
+ --keyring "$keyring" \
+ --output "$tmpOutputDir/Release" \
+ "$tmpOutputDir/InRelease"
+ [ -s "$tmpOutputDir/Release" ]
+elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then
+ rm -f "$tmpOutputDir/InRelease" # remove wget leftovers
+ gpgv \
+ --keyring "$keyring" \
+ "$tmpOutputDir/Release.gpg" \
+ "$tmpOutputDir/Release"
+ [ -s "$tmpOutputDir/Release" ]
+else
+ echo >&2 "error: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')"
+ exit 1
+fi
+
+initArgs+=(
+ # disable merged-usr (for now?) due to the following compelling arguments:
+ # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc)
+ # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still)
+ # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406
+ --no-merged-usr
+)
+
+rootfsDir="$tmpDir/rootfs"
+debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "$mirror"
+
+debuerreotype-minimizing-config "$rootfsDir"
+
+# TODO do we need to update sources.list here? (security?)
+debuerreotype-apt-get "$rootfsDir" update -qq
+
+debuerreotype-recalculate-epoch "$rootfsDir"
+epoch="$(< "$rootfsDir/debuerreotype-epoch")"
+touch_epoch() {
+ while [ "$#" -gt 0 ]; do
+ local f="$1"; shift
+ touch --no-dereference --date="@$epoch" "$f"
+ done
+}
+
+aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")"
+if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then
+ debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq
+else
+ debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq
+fi
+
+# copy the rootfs to create other variants
+mkdir "$rootfsDir"-slim
+tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim
+
+# for historical reasons (related to their usefulness in debugging non-working container networking in container early days before "--network container:xxx"), Debian 10 and older non-slim images included both "ping" and "ip" above "minbase", but in 11+ (Bullseye), that will no longer be the case and we will instead be a faithful minbase again :D
+epoch2021="$(date --date '2021-01-01 00:00:00' +%s)"
+if [ "$epoch" -lt "$epoch2021" ] || { isDebianBusterOrOlder="$([ -f "$rootfsDir/etc/os-release" ] && source "$rootfsDir/etc/os-release" && [ -n "${VERSION_ID:-}" ] && [ "${VERSION_ID%%.*}" -le 10 ] && echo 1)" && [ -n "$isDebianBusterOrOlder" ]; }; then
+ # prefer iproute2 if it exists
+ iproute=iproute2
+ if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then
+ # poor wheezy
+ iproute=iproute
+ fi
+ ping=iputils-ping
+ noInstallRecommends='--no-install-recommends'
+ debuerreotype-apt-get "$rootfsDir" install -y $noInstallRecommends $ping $iproute
+fi
+
+debuerreotype-slimify "$rootfsDir"-slim
+
+create_artifacts() {
+ local targetBase="$1"; shift
+ local rootfs="$1"; shift
+ local suite="$1"; shift
+ local variant="$1"; shift
+
+ local tarArgs=()
+
+ debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz"
+ du -hsx "$targetBase.tar.xz"
+
+ sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256"
+ touch_epoch "$targetBase.tar.xz.sha256"
+
+ debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest"
+ echo "$suite" > "$targetBase.apt-dist"
+ echo "$dpkgArch" > "$targetBase.dpkg-arch"
+ echo "$epoch" > "$targetBase.debuerreotype-epoch"
+ echo "$variant" > "$targetBase.debuerreotype-variant"
+ debuerreotype-version > "$targetBase.debuerreotype-version"
+ touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*}
+
+ for f in debian_version os-release apt/sources.list; do
+ targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")"
+ if [ -e "$rootfs/etc/$f" ]; then
+ cp "$rootfs/etc/$f" "$targetFile"
+ touch_epoch "$targetFile"
+ fi
+ done
+}
+
+for rootfs in "$rootfsDir"*/; do
+ rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ...
+
+ du -hsx "$rootfs"
+
+ variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ...
+ variant="${variant#rootfs}" # "", "-slim", ...
+ variant="${variant#-}" # "", "slim", ...
+
+ variantDir="$tmpOutputDir/$variant"
+ mkdir -p "$variantDir"
+
+ targetBase="$variantDir/rootfs"
+
+ create_artifacts "$targetBase" "$rootfs" "$suite" "$variant"
+done
+
+user="$(stat --format '%u' "$outputDir")"
+group="$(stat --format '%g' "$outputDir")"
+tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir"
diff --git a/examples/steamos.sh b/examples/steamos.sh
new file mode 100755
index 0000000..2d6ee21
--- /dev/null
+++ b/examples/steamos.sh
@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# # https://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/?C=M&O=D
+# RUN wget -O valve.deb 'https://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/valve-archive-keyring_0.6+bsosc2_all.deb' \
+# && apt-get install -y ./valve.deb \
+# && rm valve.deb
+
+debuerreotypeScriptsDir="$(which debuerreotype-init)"
+debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")"
+debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")"
+
+source "$debuerreotypeScriptsDir/.constants.sh" \
+ --flags 'arch:' \
+ -- \
+ '[--arch=<arch>] <output-dir> <suite>' \
+ 'output'
+
+eval "$dgetopt"
+arch=
+while true; do
+ flag="$1"; shift
+ dgetopt-case "$flag"
+ case "$flag" in
+ --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init
+ --) break ;;
+ *) eusage "unknown flag '$flag'" ;;
+ esac
+done
+
+outputDir="${1:-}"; shift || eusage 'missing output-dir'
+suite="${1:-brewmaster}" # http://repo.steampowered.com/steamos/dists/
+
+set -x
+
+outputDir="$(readlink -ve "$outputDir")"
+
+tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")"
+trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT
+
+export TZ='UTC' LC_ALL='C'
+
+dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}"
+
+exportDir="$tmpDir/output"
+archDir="$exportDir/steamos/$dpkgArch"
+tmpOutputDir="$archDir/$suite"
+
+mirror='http://repo.steampowered.com/steamos'
+
+initArgs=(
+ --arch "$dpkgArch"
+ --non-debian
+
+ --debootstrap-script jessie
+ --include valve-archive-keyring
+ --exclude debian-archive-keyring
+)
+
+keyring='/usr/share/keyrings/valve-archive-keyring.gpg'
+if [ -f "$keyring" ]; then
+ initArgs+=( --keyring "$keyring" )
+else
+ initArgs+=( --no-check-gpg )
+fi
+
+mkdir -p "$tmpOutputDir"
+
+if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then
+ gpgv \
+ --keyring "$keyring" \
+ --output "$tmpOutputDir/Release" \
+ "$tmpOutputDir/InRelease"
+ [ -s "$tmpOutputDir/Release" ]
+elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then
+ rm -f "$tmpOutputDir/InRelease" # remove wget leftovers
+ gpgv \
+ --keyring "$keyring" \
+ "$tmpOutputDir/Release.gpg" \
+ "$tmpOutputDir/Release"
+ [ -s "$tmpOutputDir/Release" ]
+else
+ rm -f "$tmpOutputDir/InRelease" "$tmpOutputDir/Release.gpg" "$tmpOutputDir/Release" # remove wget leftovers
+ echo >&2 "warning: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')"
+fi
+
+initArgs+=(
+ # disable merged-usr (for now?) due to the following compelling arguments:
+ # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc)
+ # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still)
+ # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406
+ --no-merged-usr
+)
+
+rootfsDir="$tmpDir/rootfs"
+debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "$mirror"
+
+debuerreotype-minimizing-config "$rootfsDir"
+
+echo "deb $mirror $suite main contrib non-free" | tee "$rootfsDir/etc/apt/sources.list"
+debuerreotype-apt-get "$rootfsDir" update -qq
+
+debuerreotype-recalculate-epoch "$rootfsDir"
+epoch="$(< "$rootfsDir/debuerreotype-epoch")"
+touch_epoch() {
+ while [ "$#" -gt 0 ]; do
+ local f="$1"; shift
+ touch --no-dereference --date="@$epoch" "$f"
+ done
+}
+touch_epoch "$rootfsDir/etc/apt/sources.list"
+
+aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")"
+if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then
+ debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq
+else
+ debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq
+fi
+
+# copy the rootfs to create other variants
+mkdir "$rootfsDir"-slim
+tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim
+
+# prefer iproute2 if it exists
+iproute=iproute2
+if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then
+ # poor wheezy
+ iproute=iproute
+fi
+debuerreotype-apt-get "$rootfsDir" install -y --no-install-recommends iputils-ping $iproute
+
+debuerreotype-slimify "$rootfsDir"-slim
+
+create_artifacts() {
+ local targetBase="$1"; shift
+ local rootfs="$1"; shift
+ local suite="$1"; shift
+ local variant="$1"; shift
+
+ local tarArgs=()
+
+ debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz"
+ du -hsx "$targetBase.tar.xz"
+
+ sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256"
+ touch_epoch "$targetBase.tar.xz.sha256"
+
+ debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest"
+ echo "$suite" > "$targetBase.apt-dist"
+ echo "$dpkgArch" > "$targetBase.dpkg-arch"
+ echo "$epoch" > "$targetBase.debuerreotype-epoch"
+ echo "$variant" > "$targetBase.debuerreotype-variant"
+ debuerreotype-version > "$targetBase.debuerreotype-version"
+ touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*}
+
+ for f in debian_version os-release apt/sources.list; do
+ targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")"
+ if [ -e "$rootfs/etc/$f" ]; then
+ cp "$rootfs/etc/$f" "$targetFile"
+ touch_epoch "$targetFile"
+ fi
+ done
+}
+
+for rootfs in "$rootfsDir"*/; do
+ rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ...
+
+ du -hsx "$rootfs"
+
+ variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ...
+ variant="${variant#rootfs}" # "", "-slim", ...
+ variant="${variant#-}" # "", "slim", ...
+
+ variantDir="$tmpOutputDir/$variant"
+ mkdir -p "$variantDir"
+
+ targetBase="$variantDir/rootfs"
+
+ create_artifacts "$targetBase" "$rootfs" "$suite" "$variant"
+done
+
+user="$(stat --format '%u' "$outputDir")"
+group="$(stat --format '%g' "$outputDir")"
+tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir"
diff --git a/examples/ubuntu.sh b/examples/ubuntu.sh
new file mode 100755
index 0000000..173f5bd
--- /dev/null
+++ b/examples/ubuntu.sh
@@ -0,0 +1,194 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# RUN apt-get update \
+# && apt-get install -y ubuntu-keyring \
+# && rm -rf /var/lib/apt/lists/*
+
+debuerreotypeScriptsDir="$(which debuerreotype-init)"
+debuerreotypeScriptsDir="$(readlink -vf "$debuerreotypeScriptsDir")"
+debuerreotypeScriptsDir="$(dirname "$debuerreotypeScriptsDir")"
+
+source "$debuerreotypeScriptsDir/.constants.sh" \
+ --flags 'arch:' \
+ -- \
+ '[--arch=<arch>] <output-dir> <suite>' \
+ 'output xenial
+--arch arm64 output bionic'
+
+eval "$dgetopt"
+arch=
+while true; do
+ flag="$1"; shift
+ dgetopt-case "$flag"
+ case "$flag" in
+ --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init
+ --) break ;;
+ *) eusage "unknown flag '$flag'" ;;
+ esac
+done
+
+outputDir="${1:-}"; shift || eusage 'missing output-dir'
+suite="${1:-}"; shift || eusage 'missing suite'
+
+set -x
+
+outputDir="$(readlink -ve "$outputDir")"
+
+tmpDir="$(mktemp --directory --tmpdir "debuerreotype.$suite.XXXXXXXXXX")"
+trap "$(printf 'rm -rf %q' "$tmpDir")" EXIT
+
+export TZ='UTC' LC_ALL='C'
+
+dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}"
+
+exportDir="$tmpDir/output"
+archDir="$exportDir/ubuntu/$dpkgArch"
+tmpOutputDir="$archDir/$suite"
+
+case "$dpkgArch" in
+ amd64 | i386)
+ mirror='http://archive.ubuntu.com/ubuntu'
+ secmirror='http://security.ubuntu.com/ubuntu'
+ ;;
+
+ *)
+ mirror='http://ports.ubuntu.com/ubuntu-ports'
+ secmirror="$mirror" # no separate security mirror for ports
+ ;;
+esac
+
+initArgs=(
+ --arch "$dpkgArch"
+ --non-debian
+)
+
+keyring='/usr/share/keyrings/ubuntu-archive-keyring.gpg'
+initArgs+=( --keyring "$keyring" )
+
+mkdir -p "$tmpOutputDir"
+
+if [ -f "$keyring" ] && wget -O "$tmpOutputDir/InRelease" "$mirror/dists/$suite/InRelease"; then
+ gpgv \
+ --keyring "$keyring" \
+ --output "$tmpOutputDir/Release" \
+ "$tmpOutputDir/InRelease"
+ [ -s "$tmpOutputDir/Release" ]
+elif [ -f "$keyring" ] && wget -O "$tmpOutputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg" && wget -O "$tmpOutputDir/Release" "$mirror/dists/$suite/Release"; then
+ rm -f "$tmpOutputDir/InRelease" # remove wget leftovers
+ gpgv \
+ --keyring "$keyring" \
+ "$tmpOutputDir/Release.gpg" \
+ "$tmpOutputDir/Release"
+ [ -s "$tmpOutputDir/Release" ]
+else
+ rm -f "$tmpOutputDir/InRelease" "$tmpOutputDir/Release.gpg" "$tmpOutputDir/Release" # remove wget leftovers
+ echo >&2 "error: failed to fetch either InRelease or Release.gpg+Release for '$suite' (from '$mirror')"
+ exit 1
+fi
+
+initArgs+=(
+ # disable merged-usr (for now?) due to the following compelling arguments:
+ # - https://bugs.debian.org/src:usrmerge ("dpkg-query" breaks, etc)
+ # - https://bugs.debian.org/914208 ("buildd" variant disables merged-usr still)
+ # - https://github.com/debuerreotype/docker-debian-artifacts/issues/60#issuecomment-461426406
+ --no-merged-usr
+)
+
+rootfsDir="$tmpDir/rootfs"
+debuerreotype-init "${initArgs[@]}" "$rootfsDir" "$suite" "$mirror"
+
+debuerreotype-minimizing-config "$rootfsDir"
+
+# setup "proper" sources.list
+tee "$rootfsDir/etc/apt/sources.list" <<-EOS
+ deb $mirror $suite main restricted universe multiverse
+ deb $mirror $suite-updates main restricted universe multiverse
+ deb $mirror $suite-backports main restricted universe multiverse
+ deb $secmirror $suite-security main restricted universe multiverse
+EOS
+# TODO make components list a script flag? backports?
+debuerreotype-apt-get "$rootfsDir" update -qq
+
+debuerreotype-recalculate-epoch "$rootfsDir"
+epoch="$(< "$rootfsDir/debuerreotype-epoch")"
+touch_epoch() {
+ while [ "$#" -gt 0 ]; do
+ local f="$1"; shift
+ touch --no-dereference --date="@$epoch" "$f"
+ done
+}
+touch_epoch "$rootfsDir/etc/apt/sources.list"
+
+aptVersion="$("$debuerreotypeScriptsDir/.apt-version.sh" "$rootfsDir")"
+if dpkg --compare-versions "$aptVersion" '>=' '1.1~'; then
+ debuerreotype-apt-get "$rootfsDir" full-upgrade -yqq
+else
+ debuerreotype-apt-get "$rootfsDir" dist-upgrade -yqq
+fi
+
+# copy the rootfs to create other variants
+mkdir "$rootfsDir"-slim
+tar -cC "$rootfsDir" . | tar -xC "$rootfsDir"-slim
+
+# prefer iproute2 if it exists
+iproute=iproute2
+if ! debuerreotype-apt-get "$rootfsDir" install -qq -s iproute2 &> /dev/null; then
+ # poor wheezy
+ iproute=iproute
+fi
+debuerreotype-apt-get "$rootfsDir" install -y --no-install-recommends iputils-ping $iproute
+
+debuerreotype-slimify "$rootfsDir"-slim
+
+create_artifacts() {
+ local targetBase="$1"; shift
+ local rootfs="$1"; shift
+ local suite="$1"; shift
+ local variant="$1"; shift
+
+ local tarArgs=()
+
+ debuerreotype-tar "${tarArgs[@]}" "$rootfs" "$targetBase.tar.xz"
+ du -hsx "$targetBase.tar.xz"
+
+ sha256sum "$targetBase.tar.xz" | cut -d' ' -f1 > "$targetBase.tar.xz.sha256"
+ touch_epoch "$targetBase.tar.xz.sha256"
+
+ debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest"
+ echo "$suite" > "$targetBase.apt-dist"
+ echo "$dpkgArch" > "$targetBase.dpkg-arch"
+ echo "$epoch" > "$targetBase.debuerreotype-epoch"
+ echo "$variant" > "$targetBase.debuerreotype-variant"
+ debuerreotype-version > "$targetBase.debuerreotype-version"
+ touch_epoch "$targetBase".{manifest,apt-dist,dpkg-arch,debuerreotype-*}
+
+ for f in debian_version os-release apt/sources.list; do
+ targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")"
+ if [ -e "$rootfs/etc/$f" ]; then
+ cp "$rootfs/etc/$f" "$targetFile"
+ touch_epoch "$targetFile"
+ fi
+ done
+}
+
+for rootfs in "$rootfsDir"*/; do
+ rootfs="${rootfs%/}" # "../rootfs", "../rootfs-slim", ...
+
+ du -hsx "$rootfs"
+
+ variant="$(basename "$rootfs")" # "rootfs", "rootfs-slim", ...
+ variant="${variant#rootfs}" # "", "-slim", ...
+ variant="${variant#-}" # "", "slim", ...
+
+ variantDir="$tmpOutputDir/$variant"
+ mkdir -p "$variantDir"
+
+ targetBase="$variantDir/rootfs"
+
+ create_artifacts "$targetBase" "$rootfs" "$suite" "$variant"
+done
+
+user="$(stat --format '%u' "$outputDir")"
+group="$(stat --format '%g' "$outputDir")"
+tar --create --directory="$exportDir" --owner="$user" --group="$group" . | tar --extract --verbose --directory="$outputDir"
diff --git a/raspbian.sh b/raspbian.sh
deleted file mode 100755
index b6bac34..0000000
--- a/raspbian.sh
+++ /dev/null
@@ -1,177 +0,0 @@
-#!/usr/bin/env bash
-set -Eeuo pipefail
-
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
-source "$thisDir/scripts/.constants.sh" \
- --flags 'no-build' \
- -- \
- '[--no-build] <output-dir> <suite>' \
- 'output stretch'
-
-eval "$dgetopt"
-build=1
-while true; do
- flag="$1"; shift
- dgetopt-case "$flag"
- case "$flag" in
- --no-build) build= ;; # for skipping "docker build"
- --) break ;;
- *) eusage "unknown flag '$flag'" ;;
- esac
-done
-
-outputDir="${1:-}"; shift || eusage 'missing output-dir'
-suite="${1:-}"; shift || eusage 'missing suite'
-
-mkdir -p "$outputDir"
-outputDir="$(readlink -f "$outputDir")"
-
-securityArgs=(
- --cap-add SYS_ADMIN
- --cap-drop SETFCAP
-)
-if docker info | grep -q apparmor; then
- # AppArmor blocks mount :)
- securityArgs+=(
- --security-opt apparmor=unconfined
- )
-fi
-
-ver="$("$thisDir/scripts/debuerreotype-version")"
-ver="${ver%% *}"
-dockerImage="debuerreotype/debuerreotype:$ver"
-[ -z "$build" ] || docker build -t "$dockerImage" "$thisDir"
-
-raspbianDockerImage="$dockerImage-raspbian"
-[ -z "$build" ] || docker build -t "$raspbianDockerImage" - <<-EODF
- FROM $dockerImage
- RUN wget -O raspbian.deb 'https://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring-udeb_20120528.2_all.udeb' \\
- && apt install -y ./raspbian.deb \\
- && rm raspbian.deb
-EODF
-
-docker run \
- --rm \
- "${securityArgs[@]}" \
- -v /tmp \
- -w /tmp \
- -e suite="$suite" \
- -e TZ='UTC' -e LC_ALL='C' \
- "$raspbianDockerImage" \
- bash -Eeuo pipefail -c '
- set -x
-
- mirror="http://archive.raspbian.org/raspbian"
-
- dpkgArch="armhf"
-
- exportDir="output"
- outputDir="$exportDir/raspbian/$dpkgArch/$suite"
-
- mkdir -p "$outputDir"
- wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg"
- wget -O "$outputDir/Release" "$mirror/dists/$suite/Release"
- gpgv \
- --keyring /usr/share/keyrings/raspbian-archive-keyring.gpg \
- "$outputDir/Release.gpg" \
- "$outputDir/Release"
-
- {
- debuerreotype-init --non-debian \
- --arch "$dpkgArch" \
- --keyring /usr/share/keyrings/raspbian-archive-keyring.gpg \
- --no-merged-usr \
- rootfs "$suite" "$mirror"
-
- epoch="$(< rootfs/debuerreotype-epoch)"
- touch_epoch() {
- while [ "$#" -gt 0 ]; do
- local f="$1"; shift
- touch --no-dereference --date="@$epoch" "$f"
- done
- }
-
- debuerreotype-minimizing-config rootfs
- debuerreotype-apt-get rootfs update -qq
- debuerreotype-apt-get rootfs dist-upgrade -yqq
-
- # make a couple copies of rootfs so we can create other variants
- for variant in slim sbuild; do
- mkdir "rootfs-$variant"
- tar -cC rootfs . | tar -xC "rootfs-$variant"
- done
-
- # prefer iproute2 if it exists
- iproute=iproute2
- if ! debuerreotype-chroot rootfs apt-get install -qq -s iproute2 &> /dev/null; then
- # poor wheezy
- iproute=iproute
- fi
- debuerreotype-apt-get rootfs install -y --no-install-recommends iputils-ping $iproute
-
- debuerreotype-slimify rootfs-slim
-
- # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild
- # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26
- # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194
- debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot
-
- create_artifacts() {
- local targetBase="$1"; shift
- local rootfs="$1"; shift
- local suite="$1"; shift
- local variant="$1"; shift
-
- if [ "$variant" != "sbuild" ]; then
- debuerreotype-tar "$rootfs" "$targetBase.tar.xz"
- else
- # sbuild needs "deb-src" entries
- debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list
-
- # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes
- # (which are used in sbuild for "--extra-package")
- # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
- # ...
- # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
- rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes"
- # TODO figure out the bug and fix it in APT instead /o\
-
- # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar")
- # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521
- debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz"
- fi
- du -hsx "$targetBase.tar.xz"
-
- sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256"
- touch_epoch "$targetBase.tar.xz.sha256"
-
- debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest"
- echo "$epoch" > "$targetBase.debuerreotype-epoch"
- touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch"
-
- for f in debian_version os-release apt/sources.list; do
- targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")"
- cp "$rootfs/etc/$f" "$targetFile"
- touch_epoch "$targetFile"
- done
- }
-
- for rootfs in rootfs*/; do
- rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ...
-
- du -hsx "$rootfs"
-
- variant="${rootfs#rootfs}" # "", "-slim", ...
- variant="${variant#-}" # "", "slim", ...
-
- variantDir="$outputDir/$variant"
- mkdir -p "$variantDir"
-
- targetBase="$variantDir/rootfs"
-
- create_artifacts "$targetBase" "$rootfs" "$suite" "$variant"
- done
- } >&2
-
- tar -cC "$exportDir" .
- ' | tar -xvC "$outputDir"
diff --git a/scripts/.apt-version.sh b/scripts/.apt-version.sh
index 30dffd3..e366d13 100755
--- a/scripts/.apt-version.sh
+++ b/scripts/.apt-version.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'<target-dir>' \
'rootfs'
@@ -19,12 +19,26 @@ done
targetDir="${1:-}"; shift || eusage 'missing target-dir'
[ -n "$targetDir" ]
-# scrape our APT version so we can do some basic feature detection (especially to remove unsupported settings on --debian-eol)
+package="${1:-apt}"
+
+# if dpkg-query does not exist, we must be on woody or older, so just assume something ancient (suggested version is the one in woody, since it should be old enough for any "fancy" features we're using this to exclude)
+fallback=
+case "$package" in
+ apt) fallback='0.5.4' ;; # woody
+ dpkg) fallback='1.9.21' ;; # woody
+esac
+
+# scrape package versions so we can do some basic feature detection (especially to remove unsupported settings on --debian-eol)
"$thisDir/debuerreotype-chroot" "$targetDir" bash -c '
+ package="$1"; shift
+ fallback="$1"; shift
if command -v dpkg-query &> /dev/null; then
- dpkg-query --show --showformat "\${Version}\n" apt
+ dpkg-query --show --showformat "\${Version}\n" "$package"
+ elif [ -n "$fallback" ]; then
+ # if dpkg-query does not exist, we must be on woody or older
+ echo "$fallback"
else
- # if dpkg-query does not exist, we must be on woody or potato, so just assume something ancient like 0.5.4 (since that is what woody includes and is old enough to cover all our features being excluded)
- echo 0.5.4
+ echo >&2 "error: missing dpkg-query and no fallback defined in debuerreotype for $package"
+ exit 1
fi
-'
+' -- "$package" "$fallback"
diff --git a/scripts/.constants.sh b/scripts/.constants.sh
index 4dacb39..eabd1ab 100644
--- a/scripts/.constants.sh
+++ b/scripts/.constants.sh
@@ -3,7 +3,7 @@
# constants of the universe
export TZ='UTC' LC_ALL='C'
umask 0002
-scriptsDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+scriptsDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
self="$(basename "$0")"
options="$(getopt -n "$BASH_SOURCE" -o '+' --long 'flags:,flags-short:' -- "$@")"
diff --git a/scripts/.debian-mirror.sh b/scripts/.debian-mirror.sh
new file mode 100755
index 0000000..8d60838
--- /dev/null
+++ b/scripts/.debian-mirror.sh
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
+source "$thisDir/.constants.sh" \
+ --flags 'eol,ports' \
+ -- \
+ '[--eol] [--ports] <timestamp> <suite> <arch> <component>' \
+ '--eol 2021-03-01T00:00:00Z jessie amd64 main
+2021-03-01T00:00:00Z buster-security arm64 main
+--ports 2021-03-01T00:00:00Z sid riscv64 main'
+
+eval "$dgetopt"
+eol=
+ports=
+while true; do
+ flag="$1"; shift
+ dgetopt-case "$flag"
+ case "$flag" in
+ --eol) eol=1 ;;
+ --ports) ports=1 ;;
+ --) break ;;
+ *) eusage "unknown flag '$flag'" ;;
+ esac
+done
+
+timestamp="${1:-}"; shift || eusage 'missing timestamp'
+suite="${1:-}"; shift || eusage 'missing suite'
+arch="${1:-}"; shift || eusage 'missing arch'
+component="${1:-}"; shift || eusage 'missing component'
+
+if [[ "$suite" == *-security ]]; then
+ target='security'
+else
+ target='standard'
+fi
+
+epoch="$(date --date "$timestamp" '+%s')"
+
+if [ -z "$ports" ]; then
+ standardMirrors=( 'http://deb.debian.org/debian' )
+ snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch")" )
+else
+ standardMirrors=( 'http://deb.debian.org/debian-ports' )
+ snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-ports')" )
+fi
+
+securityMirrors=( 'http://security.debian.org/debian-security' )
+snapshotSecurityMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-security')" )
+
+if [ -n "$eol" ]; then
+ archiveSnapshotMirror="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-archive')"
+
+ standardMirrors=( 'http://archive.debian.org/debian' "${standardMirrors[@]}" )
+ snapshotStandardMirrors=( "$archiveSnapshotMirror/debian" "${snapshotStandardMirrors[@]}" )
+
+ securityMirrors=( 'http://archive.debian.org/debian-security' "${securityMirrors[@]}" )
+ snapshotSecurityMirrors=( "$archiveSnapshotMirror/debian-security" "${snapshotSecurityMirrors[@]}" )
+fi
+
+case "$target" in
+ standard)
+ nonSnapshotMirrors=( "${standardMirrors[@]}" )
+ snapshotMirrors=( "${snapshotStandardMirrors[@]}" )
+ ;;
+ security)
+ nonSnapshotMirrors=( "${securityMirrors[@]}" )
+ snapshotMirrors=( "${snapshotSecurityMirrors[@]}" )
+ ;;
+ *) echo >&2 "error: unknown target: '$target'"; exit 1 ;;
+esac
+
+_find() {
+ local findSuite="${1:-$suite}"
+ local i
+ for i in "${!snapshotMirrors[@]}"; do
+ local mirror="${nonSnapshotMirrors[$i]}" snapshotMirror="${snapshotMirrors[$i]}"
+ # http://snapshot.debian.org/archive/debian-archive/20160314T000000Z/debian/dists/squeeze-updates/main/binary-amd64/Packages.gz
+ if \
+ wget --quiet --spider -O /dev/null -o /dev/null "$snapshotMirror/dists/$findSuite/$component/binary-$arch/Packages.xz" \
+ || wget --quiet --spider -O /dev/null -o /dev/null "$snapshotMirror/dists/$findSuite/$component/binary-$arch/Packages.gz" \
+ ; then
+ declare -g mirror="$mirror" snapshotMirror="$snapshotMirror" suite="$findSuite"
+ return 0
+ fi
+ done
+ if [ "$target" = 'security' ] && [[ "$findSuite" == *-security ]]; then
+ if _find "${suite%-security}/updates"; then
+ return 0
+ fi
+ fi
+ return 1
+}
+if ! _find; then
+ echo >&2 "warning: no apparent '$suite/$component' for '$arch' on any of the following"
+ for mirror in "${snapshotMirrors[@]}"; do echo >&2 " - $mirror"; done
+ exit 1
+fi
+printf 'mirror=%q\nsnapshotMirror=%q\nfoundSuite=%q\n' "$mirror" "$snapshotMirror" "$suite"
diff --git a/scripts/.debootstrap-scripts/potato b/scripts/.debootstrap-scripts/potato
new file mode 100644
index 0000000..110a415
--- /dev/null
+++ b/scripts/.debootstrap-scripts/potato
@@ -0,0 +1,124 @@
+#
+# This script was taken from debootstrap version 1.0.125 and then hacked to avoid invoking "init u" in postinst scripts (which tries to send SIGHUP to PID 1).
+#
+
+mirror_style release
+download_style apt var-state
+default_mirror http://archive.debian.org/debian
+force_md5
+
+LIBC=libc6
+if [ "$ARCH" = alpha ]; then
+ LIBC="libc6.1"
+fi
+
+work_out_debs () {
+ required="base-files base-passwd bash bsdutils debconf-tiny debianutils diff dpkg e2fsprogs fileutils findutils grep gzip hostname ldso libc6 libdb2 libgdbmg1 libncurses5 libnewt0 libpam-modules libpam-runtime libpam0g libpopt0 libreadline4 libstdc++2.10 login makedev mawk modutils mount ncurses-base ncurses-bin passwd perl-5.005-base perl-base procps sed shellutils slang1 sysklogd sysvinit tar textutils update util-linux whiptail"
+
+ base="adduser ae apt base-config elvis-tiny fbset fdutils gettext-base console-data console-tools console-tools-libs libdb2 libwrap0 locales modconf netbase ftp ppp pppconfig pump tasksel tcpd textutils telnet xviddetect"
+
+ without_package () {
+ echo "$2" | tr ' ' '\n' | grep -v "^$1$" | tr '\n' ' '
+ }
+
+ case $ARCH in
+ "alpha")
+ required="$(without_package "libc6" "$required") libc6.1"
+ ;;
+ "i386")
+ base="$base fdflush isapnptools lilo mbr pciutils pcmcia-cs psmisc setserial syslinux"
+ ;;
+ *)
+ # other arches may have special needs not yet represented here
+ # oh well, Potato is old
+ esac
+}
+
+first_stage_install () {
+ extract $required
+
+ :> "$TARGET/var/lib/dpkg/status"
+ echo > "$TARGET/var/lib/dpkg/available"
+
+ setup_etc
+ echo '# UNCONFIGURED FSTAB FOR BASE SYSTEM' > "$TARGET/etc/fstab"
+ chown 0:0 "$TARGET/etc/fstab"; chmod 644 "$TARGET/etc/fstab"
+
+ x_feign_install () {
+ local pkg=$1
+ local deb="$(debfor $pkg)"
+ local ver="$(extract_deb_field "$TARGET/$deb" Version)"
+
+ mkdir -p "$TARGET/var/lib/dpkg/info"
+
+echo \
+"Package: $pkg
+Version: $ver
+Status: install ok installed" >> "$TARGET/var/lib/dpkg/status"
+
+ touch "$TARGET/var/lib/dpkg/info/${pkg}.list"
+ }
+
+ setup_devices
+
+ x_feign_install dpkg
+
+ if [ -e "$TARGET/usr/bin/perl-5.005.dist" ]; then
+ mv "$TARGET/usr/bin/perl-5.005.dist" "$TARGET/usr/bin/perl-5.005"
+ fi
+ if [ ! -e "$TARGET/usr/bin/perl" ]; then
+ ln -sf perl-5.005 "$TARGET/usr/bin/perl"
+ fi
+}
+
+second_stage_install () {
+ x_core_install () {
+ in_target dpkg --force-depends --install $(debfor "$@")
+ }
+
+ export DEBIAN_FRONTEND=Noninteractive
+
+ setup_proc
+ ln "$TARGET/sbin/ldconfig.new" "$TARGET/sbin/ldconfig"
+ in_target /sbin/ldconfig
+
+ x_core_install base-files base-passwd ldso
+ x_core_install dpkg
+
+ ln -sf /usr/share/zoneinfo/UTC "$TARGET/etc/localtime"
+
+ # the libc6 and sysvinit postinst scripts want to run "init u" to re-exec sysvinit, but that sends SIGHUP to PID 1, which is undesirable (for hopefully obvious reasons), so we have to get a little bit clever while installing libc6 and later before configuring the rest of the system
+ x_hack_avoid_postinst_init() {
+ local postinst
+ for postinst in "$TARGET"/var/lib/dpkg/info/*.postinst; do
+ if grep -qE '^[[:space:]]*init[[:space:]]' "$postinst" 2>/dev/null; then
+ sed -ri -e 's/^([[:space:]]*)(init[[:space:]])/\1: # \2/' "$postinst"
+ fi
+ done
+ }
+ in_target dpkg --force-depends --unpack $(debfor $LIBC)
+ x_hack_avoid_postinst_init
+ in_target dpkg --force-depends --configure $LIBC
+
+ smallyes '' | x_core_install perl-5.005-base
+ x_core_install mawk
+ x_core_install debconf-tiny
+
+ in_target dpkg-preconfigure $(debfor $required $base)
+
+ repeatn 5 in_target dpkg --force-depends --unpack $(debfor $required)
+
+ # see above
+ x_hack_avoid_postinst_init
+
+ mv "$TARGET/sbin/start-stop-daemon" "$TARGET/sbin/start-stop-daemon.REAL"
+ cp "$TARGET/bin/true" "$TARGET/sbin/start-stop-daemon"
+
+ setup_dselect_method apt
+
+ in_target dpkg --configure --pending --force-configure-any --force-depends
+
+ smallyes '' | repeatn 5 in_target dpkg --force-auto-select --force-overwrite --skip-same-version --install $(debfor $base)
+
+ mv "$TARGET/sbin/start-stop-daemon.REAL" "$TARGET/sbin/start-stop-daemon"
+}
diff --git a/scripts/.debootstrap-scripts/slink b/scripts/.debootstrap-scripts/slink
new file mode 100644
index 0000000..a6c6a43
--- /dev/null
+++ b/scripts/.debootstrap-scripts/slink
@@ -0,0 +1,101 @@
+#
+# This script was taken from debootstrap version 0.2.45-0.2, adapted with several changes to match the corresponding changes to "potato" since it was removed, and then hacked to avoid invoking "init u" in postinst scripts (which tries to send SIGHUP to PID 1).
+#
+
+mirror_style main
+download_style apt var-state
+default_mirror http://archive.debian.org/debian
+force_md5
+
+work_out_debs () {
+ required="base-files base-passwd bash bsdutils debianutils diff dpkg e2fsprogs fileutils findutils grep gzip hostname ldso libdb2 libgdbmg1 libncurses4 ncurses3.4 libpam0g libpam0g-util libpwdb0g libreadlineg2 libstdc++2.9 login makedev mawk modutils mount ncurses-base ncurses-bin newt0.25 passwd perl-base procps sed shellutils slang1 sysklogd sysvinit tar textutils update util-linux whiptail"
+
+ base="adduser ae apt elvis-tiny fbset fdutils console-tools console-tools-libs libdb2 locales modconf netbase textutils telnet"
+
+ case $ARCH in
+ "i386")
+ required="$required libc6"
+ base="$base fdflush isapnptools lilo mbr pciutils psmisc setserial syslinux"
+ ;;
+ esac
+}
+
+first_stage_install () {
+ extract $required
+
+ :> "$TARGET/var/lib/dpkg/status"
+ echo > "$TARGET/var/lib/dpkg/available"
+
+ setup_etc
+ echo '# UNCONFIGURED FSTAB FOR BASE SYSTEM' > "$TARGET/etc/fstab"
+ chown 0:0 "$TARGET/etc/fstab"; chmod 644 "$TARGET/etc/fstab"
+
+ x_feign_install () {
+ local pkg=$1
+ local deb="$(debfor $pkg)"
+ local ver="$(extract_deb_field "$TARGET/$deb" Version)"
+
+ mkdir -p "$TARGET/var/lib/dpkg/info"
+
+echo \
+"Package: $pkg
+Version: $ver
+Status: install ok installed" >> "$TARGET/var/lib/dpkg/status"
+
+ touch "$TARGET/var/lib/dpkg/info/${pkg}.list"
+ }
+
+ setup_devices
+
+ x_feign_install dpkg
+
+ mv "$TARGET/usr/bin/perl.dist" "$TARGET/usr/bin/perl"
+}
+
+second_stage_install () {
+ x_core_install () {
+ in_target dpkg --force-depends --install $(debfor "$@")
+ }
+
+ setup_proc
+ ln "$TARGET/sbin/ldconfig.new" "$TARGET/sbin/ldconfig"
+ in_target /sbin/ldconfig
+
+ x_core_install base-files base-passwd ldso
+ x_core_install dpkg
+
+ ln -sf /usr/share/zoneinfo/UTC "$TARGET/etc/localtime"
+
+ # the libc6 and sysvinit postinst scripts want to run "init u" to re-exec sysvinit, but that sends SIGHUP to PID 1, which is undesirable (for hopefully obvious reasons), so we have to get a little bit clever while installing libc6 and later before configuring the rest of the system
+ x_hack_avoid_postinst_init() {
+ local postinst
+ for postinst in "$TARGET"/var/lib/dpkg/info/*.postinst; do
+ if grep -qE '^[[:space:]]*init[[:space:]]' "$postinst" 2>/dev/null; then
+ sed -ri -e 's/^([[:space:]]*)(init[[:space:]])/\1: # \2/' "$postinst"
+ fi
+ done
+ }
+ in_target dpkg --force-depends --unpack $(debfor libc6)
+ x_hack_avoid_postinst_init
+ in_target dpkg --force-depends --configure libc6
+
+ x_core_install perl-base
+ x_core_install mawk
+
+ repeatn 5 in_target dpkg --force-depends --unpack $(debfor $required)
+
+ # see above
+ x_hack_avoid_postinst_init
+
+ mv "$TARGET/sbin/start-stop-daemon" "$TARGET/sbin/start-stop-daemon.REAL"
+ cp "$TARGET/bin/true" "$TARGET/sbin/start-stop-daemon"
+
+ setup_dselect_method apt
+
+ in_target dpkg --configure --pending --force-configure-any --force-depends
+
+ smallyes '' | repeatn 5 in_target dpkg --force-auto-select --force-overwrite \
+ --skip-same-version --install $(debfor $base)
+
+ mv "$TARGET/sbin/start-stop-daemon.REAL" "$TARGET/sbin/start-stop-daemon"
+}
diff --git a/scripts/.dpkg-arch.sh b/scripts/.dpkg-arch.sh
index 6d7a33b..ec67f7f 100755
--- a/scripts/.dpkg-arch.sh
+++ b/scripts/.dpkg-arch.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'<target-dir>' \
'rootfs'
@@ -19,6 +19,15 @@ done
targetDir="${1:-}"; shift || eusage 'missing target-dir'
[ -n "$targetDir" ]
+if [ -s "$targetDir/etc/debian_version" ] && debVer="$(< "$targetDir/etc/debian_version")" && [ "$debVer" = '2.1' ]; then
+ # must be slink, where invoking "dpkg --print-architecture" leads to:
+ # dpkg (subprocess): failed to exec C compiler `gcc': No such file or directory
+ # dpkg: subprocess gcc --print-libgcc-file-name returned error exit status 2
+ echo 'i386'
+ # (we don't support any of "alpha", "m68k", or "sparc"; see http://archive.debian.org/debian/dists/slink/ -- if we ever do, "apt-get --version" is a good candidate for scraping: "apt 0.3.11 for i386 compiled on Aug 8 1999 10:12:36")
+ exit
+fi
+
arch="$("$thisDir/debuerreotype-chroot" "$targetDir" dpkg --print-architecture)"
# --debian-eol woody likes to give us "i386-none"
diff --git a/scripts/.fix-apt-comments.sh b/scripts/.fix-apt-comments.sh
index 1e8b545..6b4adfd 100755
--- a/scripts/.fix-apt-comments.sh
+++ b/scripts/.fix-apt-comments.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'<apt-version> <file> [file ...]' \
'0.7.22 rootfs/etc/apt/apt.conf.d/example'
diff --git a/scripts/.gpgv-ignore-expiration.sh b/scripts/.gpgv-ignore-expiration.sh
index 31690bf..396bf62 100755
--- a/scripts/.gpgv-ignore-expiration.sh
+++ b/scripts/.gpgv-ignore-expiration.sh
@@ -28,7 +28,8 @@ _status_fd() {
if fd="$(_status_fd "$@")" && [ -n "$fd" ]; then
# older bash (3.2, lenny) doesn't support variable file descriptors (hence "eval")
# (bash: syntax error near unexpected token `$fd')
- eval 'exec gpgv "$@" '"$fd"'> >(sed "s/EXPKEYSIG/GOODSIG/" >&'"$fd"')'
+ sedExpression='s/^\[GNUPG:\] EXPKEYSIG /[GNUPG:] GOODSIG /'
+ eval 'exec gpgv "$@" '"$fd"'> >(sed "$sedExpression" >&'"$fd"')'
fi
# no "--status-fd"? no worries! ("gpgv" without "--status-fd" doesn't seem to care about expired keys, so we don't have to either)
diff --git a/scripts/.slimify-excludes b/scripts/.slimify-excludes
index 0b3de88..62580dc 100644
--- a/scripts/.slimify-excludes
+++ b/scripts/.slimify-excludes
@@ -3,17 +3,17 @@
# https://wiki.ubuntu.com/ReducingDiskFootprint#Drop_unnecessary_files
/usr/share/doc/*
-/usr/share/groff/*
/usr/share/info/*
/usr/share/linda/*
/usr/share/lintian/overrides/*
/usr/share/locale/*
/usr/share/man/*
+#/usr/share/groff/* (https://github.com/debuerreotype/debuerreotype/issues/87)
-# https://anonscm.debian.org/cgit/collab-maint/localepurge.git/tree/usr/share/localepurge/gen-dpkg-cfg.pl?id=b32c8d46d43be2027096f5a202ac789a637ceb39#n9
-/usr/share/locale/*
-/usr/share/gnome/help/*/*
+# https://salsa.debian.org/elmig-guest/localepurge/-/blob/176446028ca719d65993eb01e39d7040fbbcf12d/usr/share/localepurge/gen-dpkg-cfg.pl#L9-20
/usr/share/doc/kde/HTML/*/*
+/usr/share/gnome/help/*/*
+/usr/share/locale/*
/usr/share/omf/*/*-*.emf
# see also .slimify-includes
diff --git a/scripts/.slimify-includes b/scripts/.slimify-includes
index adc5c61..ee171c3 100644
--- a/scripts/.slimify-includes
+++ b/scripts/.slimify-includes
@@ -4,14 +4,14 @@
# https://wiki.ubuntu.com/ReducingDiskFootprint#Drop_unnecessary_files
/usr/share/doc/*/copyright
-# https://anonscm.debian.org/cgit/collab-maint/localepurge.git/tree/usr/share/localepurge/gen-dpkg-cfg.pl?id=b32c8d46d43be2027096f5a202ac789a637ceb39#n9
-/usr/share/locale/locale.alias
-/usr/share/gnome/help/*/C/*
+# https://salsa.debian.org/elmig-guest/localepurge/-/blob/176446028ca719d65993eb01e39d7040fbbcf12d/usr/share/localepurge/gen-dpkg-cfg.pl#L22-47
/usr/share/doc/kde/HTML/C/*
-/usr/share/omf/*/*-C.emf
-/usr/share/locale/languages
+/usr/share/gnome/help/*/C/*
/usr/share/locale/all_languages
/usr/share/locale/currency/*
/usr/share/locale/l10n/*
+/usr/share/locale/languages
+/usr/share/locale/locale.alias
+/usr/share/omf/*/*-C.emf
# see also .slimify-excludes
diff --git a/scripts/.snapshot-url.sh b/scripts/.snapshot-url.sh
index 1c0997f..d804de9 100755
--- a/scripts/.snapshot-url.sh
+++ b/scripts/.snapshot-url.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'<timestamp> [archive]' \
'2017-05-08T00:00:00Z debian-security'
diff --git a/scripts/debuerreotype-apt-get b/scripts/debuerreotype-apt-get
index a460a1a..78a1f18 100755
--- a/scripts/debuerreotype-apt-get
+++ b/scripts/debuerreotype-apt-get
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'<target-dir> arguments' \
'rootfs update'
diff --git a/scripts/debuerreotype-chroot b/scripts/debuerreotype-chroot
index ceb209b..a6f51d2 100755
--- a/scripts/debuerreotype-chroot
+++ b/scripts/debuerreotype-chroot
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'<target-dir> <command> [args...]' \
'rootfs apt-get update'
@@ -26,11 +26,13 @@ export targetDir epoch
unshare --mount bash -Eeuo pipefail -c '
[ -n "$targetDir" ] # just to be safe
for dir in dev proc sys; do
- if [ -e "$targetDir/$dir" ]; then
+ if [ -d "$targetDir/$dir" ]; then
# --debian-eol woody and below have no /sys
mount --rbind "/$dir" "$targetDir/$dir"
fi
done
- mount --rbind --read-only /etc/resolv.conf "$targetDir/etc/resolv.conf"
+ if [ -f "$targetDir/etc/resolv.conf" ]; then
+ mount --rbind --read-only /etc/resolv.conf "$targetDir/etc/resolv.conf"
+ fi
exec chroot "$targetDir" /usr/bin/env -i PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" TZ="$TZ" LC_ALL="$LC_ALL" SOURCE_DATE_EPOCH="$epoch" "$@"
' -- "$cmd" "$@"
diff --git a/scripts/debuerreotype-debian-sources-list b/scripts/debuerreotype-debian-sources-list
index 2dd77e6..ab4f679 100755
--- a/scripts/debuerreotype-debian-sources-list
+++ b/scripts/debuerreotype-debian-sources-list
@@ -1,17 +1,18 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
- --flags 'eol,snapshot' \
+ --flags 'eol,ports,snapshot' \
--flags 'deb-src' \
-- \
- '[--deb-src] [--eol] [--no-snapshot] <target-dir> <suite>' \
+ '[--deb-src] [--eol] [--ports] [--snapshot] <target-dir> <suite>' \
'--snapshot rootfs stretch
--eol rootfs wheezy'
eval "$dgetopt"
eol=
+ports=
snapshot=
debSrc=
while true; do
@@ -19,6 +20,7 @@ while true; do
dgetopt-case "$flag"
case "$flag" in
--eol) eol=1 ;;
+ --ports) ports=1 ;;
--snapshot) snapshot=1 ;;
--deb-src) debSrc=1 ;;
--) break ;;
@@ -32,19 +34,24 @@ suite="${1:-}"; shift || eusage 'missing suite'
epoch="$(< "$targetDir/debuerreotype-epoch")"
-standardMirror='http://deb.debian.org/debian'
-snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch")" )
+if [ -z "$ports" ]; then
+ standardMirrors=( 'http://deb.debian.org/debian' )
+ snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch")" )
+else
+ standardMirrors=( 'http://deb.debian.org/debian-ports' )
+ snapshotStandardMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-ports')" )
+fi
-securityMirror='http://security.debian.org/debian-security'
+securityMirrors=( 'http://security.debian.org/debian-security' )
snapshotSecurityMirrors=( "$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-security')" )
if [ -n "$eol" ]; then
archiveSnapshotMirror="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-archive')"
- standardMirror='http://archive.debian.org/debian'
+ standardMirrors=( 'http://archive.debian.org/debian' "${standardMirrors[@]}" )
snapshotStandardMirrors=( "$archiveSnapshotMirror/debian" "${snapshotStandardMirrors[@]}" )
- securityMirror='http://archive.debian.org/debian-security'
+ securityMirrors=( 'http://archive.debian.org/debian-security' "${securityMirrors[@]}" )
snapshotSecurityMirrors=( "$archiveSnapshotMirror/debian-security" "${snapshotSecurityMirrors[@]}" )
fi
@@ -54,34 +61,30 @@ arch="$("$thisDir/.dpkg-arch.sh" "$targetDir")"
deb() {
local suite="$1"; shift
local comp="$1"; shift
- local target="$1"; shift # "standard" or "security"
-
- local nonSnapshotMirror= snapshotMirrors=()
- case "$target" in
- standard) nonSnapshotMirror="$standardMirror"; snapshotMirrors=( "${snapshotStandardMirrors[@]}" ) ;;
- security) nonSnapshotMirror="$securityMirror"; snapshotMirrors=( "${snapshotSecurityMirrors[@]}" ) ;;
- *) echo >&2 "error: unknown 'deb' line target: '$target'"; exit 1 ;;
- esac
- local found= mirror
- for mirror in "${snapshotMirrors[@]}"; do
- # http://snapshot.debian.org/archive/debian-archive/20160314T000000Z/debian/dists/squeeze-updates/main/binary-amd64/Packages.gz
- if wget --quiet --spider -O /dev/null -o /dev/null "$mirror/dists/$suite/$comp/binary-$arch/Packages.gz"; then
- found="$mirror"
- break
- fi
- done
- if [ -z "$found" ]; then
- echo >&2 "warning: no apparent '$suite/$comp' for '$arch' on any of the following; skipping"
- for mirror in "${snapshotMirrors[@]}"; do echo >&2 " - $mirror"; done
+ local mirrorArgs=()
+ if [ -n "$ports" ]; then
+ mirrorArgs+=( --ports )
+ fi
+ if [ -n "$eol" ]; then
+ mirrorArgs+=( --eol )
+ fi
+ mirrorArgs+=( "@$epoch" "$suite" "$arch" "$comp" )
+ local mirrors
+ if ! mirrors="$("$thisDir/.debian-mirror.sh" "${mirrorArgs[@]}")"; then
+ echo >&2 "skipping '$suite/$comp' ..."
return
fi
+ eval "$mirrors"
+ [ -n "$mirror" ]
+ [ -n "$snapshotMirror" ]
+ [ -n "$foundSuite" ]
+ suite="$foundSuite"
if [ -n "$snapshot" ]; then
- mirror="$found"
+ mirror="$snapshotMirror"
else
- echo "# deb $found $suite $comp"
- mirror="$nonSnapshotMirror"
+ echo "# deb $snapshotMirror $suite $comp"
fi
echo "deb $mirror $suite $comp"
if [ -n "$debSrc" ]; then
@@ -92,15 +95,19 @@ deb() {
# https://github.com/tianon/go-aptsources/blob/e066ed9cd8cd9eef7198765bd00ec99679e6d0be/target.go#L16-L58
{
case "$suite" in
- sid|unstable|testing)
+ sid | unstable | testing)
deb "$suite" "$comp" standard
+ if [ -n "$ports" ]; then
+ # https://www.ports.debian.org/archive
+ deb unreleased "$comp" standard
+ fi
;;
*)
# https://salsa.debian.org/installer-team/apt-setup/tree/d7a642fb5fc76e4f0b684db53984bdb9123f8360/generators
- deb "$suite" "$comp" standard # "50mirror"
- deb "$suite/updates" "$comp" security # "91security"
- deb "$suite-updates" "$comp" standard # "92updates"
+ deb "$suite" "$comp" standard # "50mirror"
+ deb "$suite-security" "$comp" security # "91security"
+ deb "$suite-updates" "$comp" standard # "92updates"
# https://wiki.debian.org/SourcesList#Example_sources.list
if [ "$suite" = 'squeeze' ]; then
diff --git a/scripts/debuerreotype-fixup b/scripts/debuerreotype-fixup
index ddf48d5..824ddbc 100755
--- a/scripts/debuerreotype-fixup
+++ b/scripts/debuerreotype-fixup
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'<target-dir>' \
'rootfs'
@@ -21,6 +21,13 @@ targetDir="${1:-}"; shift || eusage 'missing target-dir'
epoch="$(< "$targetDir/debuerreotype-epoch")"
[ -n "$epoch" ]
+if [ -s "$targetDir/etc/machine-id" ]; then
+ # https://www.freedesktop.org/software/systemd/man/machine-id.html
+ # > For operating system images which are created once and used on multiple machines, for example for containers or in the cloud, /etc/machine-id should be either missing or an empty file in the generic file system image ...
+ echo -n > "$targetDir/etc/machine-id"
+ chmod 0644 "$targetDir/etc/machine-id"
+fi
+
# https://github.com/lamby/debootstrap/commit/66b15380814aa62ca4b5807270ac57a3c8a0558d#diff-de4eef4ab836e5c6c9c1f820a2f624baR709
rm -f \
"$targetDir/var/log/dpkg.log" \
diff --git a/scripts/debuerreotype-gpgv-ignore-expiration-config b/scripts/debuerreotype-gpgv-ignore-expiration-config
index 4de6171..416e7e8 100755
--- a/scripts/debuerreotype-gpgv-ignore-expiration-config
+++ b/scripts/debuerreotype-gpgv-ignore-expiration-config
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'<target-dir>' \
'rootfs'
diff --git a/scripts/debuerreotype-init b/scripts/debuerreotype-init
index dec401a..9aa86ca 100755
--- a/scripts/debuerreotype-init
+++ b/scripts/debuerreotype-init
@@ -1,13 +1,14 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
- --flags 'debian,debian-eol,non-debian' \
+ --flags 'debian,debian-eol,debian-ports,non-debian' \
--flags 'debootstrap:' \
--flags 'debootstrap-script:' \
--flags 'keyring:,arch:,include:,exclude:' \
--flags 'merged-usr,no-merged-usr' \
+ --flags 'check-gpg,no-check-gpg' \
-- \
'<target-dir> <suite> <timestamp>' \
'rootfs stretch 2017-05-08T00:00:00Z
@@ -19,6 +20,7 @@ source "$thisDir/.constants.sh" \
eval "$dgetopt"
nonDebian=
debianEol=
+debianPorts=
debootstrap=
script=
keyring=
@@ -26,13 +28,15 @@ arch=
include=
exclude=
noMergedUsr=
+noCheckGpg=
while true; do
flag="$1"; shift
dgetopt-case "$flag"
case "$flag" in
- --debian) nonDebian= ; debianEol= ;;
- --debian-eol) nonDebian= ; debianEol=1 ;;
- --non-debian) nonDebian=1; debianEol= ;;
+ --debian) nonDebian= ;;
+ --debian-eol) nonDebian= ; debianEol=1 ;;
+ --debian-ports) nonDebian= ; debianPorts=1 ;;
+ --non-debian) nonDebian=1 ;;
--debootstrap) debootstrap="$1"; shift ;;
--debootstrap-script) script="$1"; shift ;;
--keyring) keyring="$1"; shift ;;
@@ -41,6 +45,8 @@ while true; do
--exclude) exclude="${exclude:+$exclude,}$1"; shift ;;
--merged-usr) noMergedUsr= ;;
--no-merged-usr) noMergedUsr=1 ;;
+ --check-gpg) noCheckGpg= ;;
+ --no-check-gpg) noCheckGpg=1 ;;
--) break ;;
*) eusage "unknown flag '$flag'" ;;
esac
@@ -62,39 +68,55 @@ if [ -z "$nonDebian" ]; then
else
mirror="${1:-}"; shift || eusage 'missing mirror'
- timestamp="$(wget -qO- "$mirror/dists/$suite/Release" | awk -F ': ' '$1 == "Date" { print $2 }')"
+ timestamp="$(
+ {
+ wget -qO- "$mirror/dists/$suite/InRelease" 2>/dev/null \
+ || wget -qO- "$mirror/dists/$suite/Release" 2>/dev/null
+ } | awk -F ': ' '$1 == "Date" { print $2; exit }'
+ )" || :
+ [ -n "$timestamp" ]
+
+ # see "debuerreotype-recalculate-epoch" for a simple way to update this value appropriately after "sources.list" is updated and "apt-get update" has been run (which will be more accurate)
fi
epoch="$(date --date "$timestamp" '+%s')"
export SOURCE_DATE_EPOCH="$epoch"
if [ -z "$nonDebian" ]; then
- if [ -z "$debianEol" ]; then
- mirror="$("$thisDir/.snapshot-url.sh" "@$epoch")"
- else
- mirrorbase="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-archive')"
- mirror="$mirrorbase/debian"
+ dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- '{ print $NF }')}"
+ mirrorArgs=()
+ if [ -n "$debianPorts" ]; then
+ mirrorArgs+=( --ports )
+ fi
+ if [ -n "$debianEol" ]; then
+ mirrorArgs+=( --eol )
fi
+ mirrorArgs+=( "@$epoch" "$suite" "$dpkgArch" main )
+ mirrors="$("$thisDir/.debian-mirror.sh" "${mirrorArgs[@]}")"
+ eval "$mirrors"
+ [ -n "$snapshotMirror" ]
+ mirror="$snapshotMirror"
fi
-debootstrapArgs=(
- --force-check-gpg
-)
+debootstrapArgs=()
-minbaseSupported="$(
- scriptFile="$(
- if [ -n "$script" ]; then
- readlink -f "$script"
- else
- cd /usr/share/debootstrap/scripts
- readlink -f "$suite"
- fi
- )"
- if grep -q 'minbase' "$scriptFile"; then
- echo 1
+if [ -z "$noCheckGpg" ]; then
+ debootstrapArgs+=( --force-check-gpg )
+else
+ debootstrapArgs+=( --no-check-gpg )
+fi
+
+: "${script:=$suite}"
+script="$(
+ if [ -s "$thisDir/.debootstrap-scripts/$script" ]; then
+ readlink -vf "$thisDir/.debootstrap-scripts/$script"
+ elif [ -s "/usr/share/debootstrap/scripts/$script" ]; then
+ readlink -vf "/usr/share/debootstrap/scripts/$script"
+ else
+ readlink -vf "$script"
fi
)"
-if [ -n "$minbaseSupported" ]; then
+if grep -q 'minbase' "$script"; then
# --debian-eol sarge and older do not support minbase
debootstrapArgs+=( --variant=minbase )
fi
@@ -106,11 +128,15 @@ fi
[ -z "$exclude" ] || debootstrapArgs+=( --exclude="$exclude" )
debootstrapArgs+=(
- "$suite" "$targetDir" "$mirror"
+ "$suite" "$targetDir" "$mirror" "$script"
)
-[ -z "$script" ] || debootstrapArgs+=( "$script" )
-: "${debootstrap:=debootstrap}"
+unshare-debootstrap() {
+ # avoid bugs in packages that might leave things mounted ("/run/shm" is a common one that sometimes ends up dangling)
+ unshare --mount debootstrap "$@"
+}
+
+: "${debootstrap:=unshare-debootstrap}"
if ! "$debootstrap" "${debootstrapArgs[@]}"; then
if [ -f "$targetDir/debootstrap/debootstrap.log" ]; then
echo >&2
@@ -130,7 +156,10 @@ fi
echo "$epoch" > "$targetDir/debuerreotype-epoch"
if [ -z "$nonDebian" ]; then
- "$thisDir/debuerreotype-debian-sources-list" --snapshot $([ -z "$debianEol" ] || echo '--eol') "$targetDir" "$suite"
+ "$thisDir/debuerreotype-debian-sources-list" --snapshot \
+ $([ -z "$debianEol" ] || echo '--eol') \
+ $([ -z "$debianPorts" ] || echo '--ports') \
+ "$targetDir" "$suite"
"$thisDir/debuerreotype-apt-get" "$targetDir" update -qq
fi
@@ -150,10 +179,6 @@ fi
' -- "$include"
echo 'debuerreotype' > "$targetDir/etc/hostname"
-echo "$epoch" \
- | md5sum \
- | cut -f1 -d' ' \
- > "$targetDir/etc/machine-id" # TODO should we only do this if "/etc/machine-id" already exists?
{
echo '# https://1.1.1.1 (privacy-focused, highly-available DNS service)'
echo 'nameserver 1.1.1.1'
@@ -161,7 +186,6 @@ echo "$epoch" \
} > "$targetDir/etc/resolv.conf"
chmod 0644 \
"$targetDir/etc/hostname" \
- "$targetDir/etc/machine-id" \
"$targetDir/etc/resolv.conf"
# fix ownership/permissions on / (otherwise "debootstrap" leaves them as-is which causes reproducibility issues)
diff --git a/scripts/debuerreotype-minimizing-config b/scripts/debuerreotype-minimizing-config
index 2764e1c..2eaa7af 100755
--- a/scripts/debuerreotype-minimizing-config
+++ b/scripts/debuerreotype-minimizing-config
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'<target-dir>' \
'rootfs'
@@ -20,6 +20,7 @@ targetDir="${1:-}"; shift || eusage 'missing target-dir'
[ -n "$targetDir" ]
aptVersion="$("$thisDir/.apt-version.sh" "$targetDir")"
+dpkgVersion="$("$thisDir/.apt-version.sh" "$targetDir" 'dpkg')"
# https://github.com/docker/docker/blob/d6f4fe9e38b60f63e429fff7ffced9c26cbf8236/contrib/mkimage/debootstrap#L63-L177
@@ -43,8 +44,9 @@ if "$thisDir/debuerreotype-chroot" "$targetDir" apt-get install -qq -s upstart &
fi
# force dpkg not to call sync() after package extraction (speeding up installs)
-if [ -d "$targetDir/etc/dpkg/dpkg.cfg.d" ]; then
+if [ -d "$targetDir/etc/dpkg/dpkg.cfg.d" ] && dpkg --compare-versions "$dpkgVersion" '>=' '1.15.8.6~'; then
# --debian-eol lenny and older do not include /etc/dpkg/dpkg.cfg.d
+ # force-unsafe-io was added in dpkg 1.15.8.6: https://salsa.debian.org/dpkg-team/dpkg/-/commit/929a9c4808c79781469987585f78f07df7f1d484
cat > "$targetDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" <<-'EOF'
# For most Docker users, package installs happen during "docker build", which
# doesn't survive power loss and gets restarted clean afterwards anyhow, so
@@ -122,7 +124,7 @@ if [ -d "$targetDir/etc/apt/apt.conf.d" ]; then
EOF
# https://github.com/debuerreotype/debuerreotype/issues/41
isDebianJessie="$([ -f "$targetDir/etc/os-release" ] && source "$targetDir/etc/os-release" && [ "${ID:-}" = 'debian' ] && [ "${VERSION_ID:-}" = '8' ] && echo '1')" || :
- if [ -n "$isDebianJessie" ] || [[ "$aptVersion" == 0.* ]] || "$thisDir/debuerreotype-chroot" "$targetDir" dpkg --compare-versions "$aptVersion" '<<' '1.0.9.2~'; then
+ if [ -n "$isDebianJessie" ] || [[ "$aptVersion" == 0.* ]] || dpkg --compare-versions "$aptVersion" '<<' '1.0.9.2~'; then
cat >> "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF'
# https://salsa.debian.org/apt-team/apt/commit/b0f4b486e6850c5f98520ccf19da71d0ed748ae4; released in src:apt 1.0.9.2, 2014-10-02
diff --git a/scripts/debuerreotype-recalculate-epoch b/scripts/debuerreotype-recalculate-epoch
new file mode 100755
index 0000000..1152203
--- /dev/null
+++ b/scripts/debuerreotype-recalculate-epoch
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
+source "$thisDir/.constants.sh" \
+ '<target-dir>' \
+ 'rootfs'
+
+eval "$dgetopt"
+while true; do
+ flag="$1"; shift
+ dgetopt-case "$flag"
+ case "$flag" in
+ --) break ;;
+ *) eusage "unknown flag '$flag'" ;;
+ esac
+done
+
+targetDir="${1:-}"; shift || eusage 'missing target-dir'
+[ -n "$targetDir" ]
+
+# ideally this would use something like "apt-get indextargets" instead of hard-coding these particular "/var/lib/apt/lists" paths, but it doesn't include the Release files :(
+# also a caution from DonKult: "the Release file might really be an InRelease file which failed signature checks"
+
+shopt -s nullglob
+releaseFiles=( "$targetDir"/var/lib/apt/lists/*_{In,}Release )
+if [ "${#releaseFiles[@]}" -eq 0 ]; then
+ echo >&2 "error: no 'Release' files found at /var/lib/apt/lists in '$targetDir'"
+ echo >&2 " did you forget to populate 'sources.list' or run 'apt-get update' first?"
+ exit 1
+fi
+
+epoch="$(
+ awk -F ': ' '$1 == "Date" { printf "%s%c", $2, 0 }' "${releaseFiles[@]}" \
+ | xargs -r0n1 date '+%s' --date \
+ | sort -un \
+ | tail -1
+)"
+echo "$epoch" > "$targetDir/debuerreotype-epoch"
diff --git a/scripts/debuerreotype-slimify b/scripts/debuerreotype-slimify
index 618d916..b3fe6a1 100755
--- a/scripts/debuerreotype-slimify
+++ b/scripts/debuerreotype-slimify
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'<target-dir>' \
'rootfs'
@@ -19,6 +19,20 @@ done
targetDir="${1:-}"; shift || eusage 'missing target-dir'
[ -n "$targetDir" ]
+dpkgVersion="$("$thisDir/.apt-version.sh" "$targetDir" 'dpkg')"
+
+if ! { [ -d "$targetDir/etc/dpkg/dpkg.cfg.d" ] && dpkg --compare-versions "$dpkgVersion" '>=' '1.15.8.6~'; }; then
+ # --debian-eol lenny and older do not include /etc/dpkg/dpkg.cfg.d
+ # path-exclude/include was added in dpkg 1.15.8: https://salsa.debian.org/dpkg-team/dpkg/-/commit/4694cd64089bc72975d8ba6fbe51339023eb2e8c
+ echo >&2 "note: skipping $self: dpkg version ($dpkgVersion) too old to support path-exclude"
+ exit
+fi
+
+# https://github.com/debuerreotype/debuerreotype/issues/10
+shopt -s nullglob
+extraSpecialDirectories=( "$targetDir"/usr/share/man/man[0-9]/ )
+shopt -u nullglob
+
IFS=$'\n'; set -o noglob
slimExcludes=( $(grep -vE '^#|^$' "$thisDir/.slimify-excludes" | sort -u) )
slimIncludes=( $(grep -vE '^#|^$' "$thisDir/.slimify-includes" | sort -u) )
@@ -85,3 +99,8 @@ done
done
} >> "$dpkgCfgFile"
chmod 0644 "$dpkgCfgFile"
+
+# https://github.com/debuerreotype/debuerreotype/issues/10
+if [ "${#extraSpecialDirectories[@]}" -gt 0 ]; then
+ mkdir -p "${extraSpecialDirectories[@]}"
+fi
diff --git a/scripts/debuerreotype-tar b/scripts/debuerreotype-tar
index 80e8c99..95f0cfa 100755
--- a/scripts/debuerreotype-tar
+++ b/scripts/debuerreotype-tar
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
--flags 'exclude:' \
--flags 'include-dev' \
diff --git a/scripts/debuerreotype-version b/scripts/debuerreotype-version
index f87e9a5..a457ac3 100755
--- a/scripts/debuerreotype-version
+++ b/scripts/debuerreotype-version
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -Eeuo pipefail
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+thisDir="$(dirname "$(readlink -vf "$BASH_SOURCE")")"
source "$thisDir/.constants.sh" \
'' \
''
diff --git a/steamos.sh b/steamos.sh
deleted file mode 100755
index b40a084..0000000
--- a/steamos.sh
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/usr/bin/env bash
-set -Eeuo pipefail
-
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
-source "$thisDir/scripts/.constants.sh" \
- --flags 'no-build' \
- -- \
- '[--no-build] <output-dir> [suite]' \
- 'output'
-
-eval "$dgetopt"
-build=1
-while true; do
- flag="$1"; shift
- dgetopt-case "$flag"
- case "$flag" in
- --no-build) build= ;; # for skipping "docker build"
- --) break ;;
- *) eusage "unknown flag '$flag'" ;;
- esac
-done
-
-outputDir="${1:-}"; shift || eusage 'missing output-dir'
-suite="${1:-brewmaster}" # http://repo.steampowered.com/steamos/dists/
-
-mkdir -p "$outputDir"
-outputDir="$(readlink -f "$outputDir")"
-
-securityArgs=(
- --cap-add SYS_ADMIN
- --cap-drop SETFCAP
-)
-if docker info | grep -q apparmor; then
- # AppArmor blocks mount :)
- securityArgs+=(
- --security-opt apparmor=unconfined
- )
-fi
-
-ver="$("$thisDir/scripts/debuerreotype-version")"
-ver="${ver%% *}"
-dockerImage="debuerreotype/debuerreotype:$ver"
-[ -z "$build" ] || docker build -t "$dockerImage" "$thisDir"
-
-steamDockerImage="$dockerImage-steamos"
-[ -z "$build" ] || docker build -t "$steamDockerImage" - <<-EODF
- FROM $dockerImage
- # http://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/?C=M;O=D
- RUN wget -O valve.deb 'http://repo.steampowered.com/steamos/pool/main/v/valve-archive-keyring/valve-archive-keyring_0.5+bsos3_all.deb' \\
- && apt install -y ./valve.deb \\
- && rm valve.deb
-EODF
-
-docker run \
- --rm \
- "${securityArgs[@]}" \
- --tmpfs /tmp:dev,exec,suid,noatime \
- -w /tmp \
- -e suite="$suite" \
- -e TZ='UTC' -e LC_ALL='C' \
- "$steamDockerImage" \
- bash -Eeuo pipefail -c '
- set -x
-
- mirror="http://repo.steampowered.com/steamos"
-
- dpkgArch="$(dpkg --print-architecture | awk -F- "{ print \$NF }")"
-
- exportDir="output"
- outputDir="$exportDir/steamos/$dpkgArch/$suite"
-
- debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")"
-
- mkdir -p "$outputDir"
- wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg"
- wget -O "$outputDir/Release" "$mirror/dists/$suite/Release"
- gpgv \
- --keyring /usr/share/keyrings/valve-archive-keyring.gpg \
- "$outputDir/Release.gpg" \
- "$outputDir/Release"
-
- {
- debuerreotype-init --non-debian \
- --debootstrap-script /usr/share/debootstrap/scripts/jessie \
- --keyring /usr/share/keyrings/valve-archive-keyring.gpg \
- --include valve-archive-keyring \
- --exclude debian-archive-keyring \
- --no-merged-usr \
- rootfs "$suite" "$mirror"
- echo "deb $mirror $suite main contrib non-free" | tee rootfs/etc/apt/sources.list
-
- epoch="$(< rootfs/debuerreotype-epoch)"
- touch_epoch() {
- while [ "$#" -gt 0 ]; do
- local f="$1"; shift
- touch --no-dereference --date="@$epoch" "$f"
- done
- }
-
- debuerreotype-minimizing-config rootfs
- debuerreotype-apt-get rootfs update -qq
- debuerreotype-apt-get rootfs dist-upgrade -yqq
-
- # make a couple copies of rootfs so we can create other variants
- for variant in slim sbuild; do
- mkdir "rootfs-$variant"
- tar -cC rootfs . | tar -xC "rootfs-$variant"
- done
-
- # prefer iproute2 if it exists
- iproute=iproute2
- if ! debuerreotype-chroot rootfs apt-get install -qq -s iproute2 &> /dev/null; then
- # poor wheezy
- iproute=iproute
- fi
- debuerreotype-apt-get rootfs install -y --no-install-recommends iputils-ping $iproute
-
- debuerreotype-slimify rootfs-slim
-
- # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild
- # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26
- # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194
- fakeroot=fakeroot
- if [[ "$suite" == alchemist* ]]; then
- # poor alchemist
- fakeroot=
- fi
- debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential $fakeroot
-
- create_artifacts() {
- local targetBase="$1"; shift
- local rootfs="$1"; shift
- local suite="$1"; shift
- local variant="$1"; shift
-
- if [ "$variant" != "sbuild" ]; then
- debuerreotype-tar "$rootfs" "$targetBase.tar.xz"
- else
- # sbuild needs "deb-src" entries
- debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list
-
- # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes
- # (which are used in sbuild for "--extra-package")
- # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
- # ...
- # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
- rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes"
- # TODO figure out the bug and fix it in APT instead /o\
-
- # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar")
- # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521
- debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz"
- fi
- du -hsx "$targetBase.tar.xz"
-
- sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256"
- touch_epoch "$targetBase.tar.xz.sha256"
-
- debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest"
- echo "$epoch" > "$targetBase.debuerreotype-epoch"
- touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch"
-
- for f in debian_version os-release apt/sources.list; do
- targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")"
- cp "$rootfs/etc/$f" "$targetFile"
- touch_epoch "$targetFile"
- done
- }
-
- for rootfs in rootfs*/; do
- rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ...
-
- du -hsx "$rootfs"
-
- variant="${rootfs#rootfs}" # "", "-slim", ...
- variant="${variant#-}" # "", "slim", ...
-
- variantDir="$outputDir/$variant"
- mkdir -p "$variantDir"
-
- targetBase="$variantDir/rootfs"
-
- create_artifacts "$targetBase" "$rootfs" "$suite" "$variant"
- done
- } >&2
-
- tar -cC "$exportDir" .
- ' | tar -xvC "$outputDir"
diff --git a/ubuntu.sh b/ubuntu.sh
deleted file mode 100755
index cb1a3cb..0000000
--- a/ubuntu.sh
+++ /dev/null
@@ -1,192 +0,0 @@
-#!/usr/bin/env bash
-set -Eeuo pipefail
-
-thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
-source "$thisDir/scripts/.constants.sh" \
- --flags 'no-build' \
- --flags 'arch:' \
- -- \
- '[--no-build] [--arch=<arch>] <output-dir> <suite>' \
- 'output xenial
---arch arm64 output bionic'
-
-eval "$dgetopt"
-build=1
-arch=
-while true; do
- flag="$1"; shift
- dgetopt-case "$flag"
- case "$flag" in
- --no-build) build= ;; # for skipping "docker build"
- --arch) arch="$1"; shift ;; # for adding "--arch" to debuerreotype-init
- --) break ;;
- *) eusage "unknown flag '$flag'" ;;
- esac
-done
-
-outputDir="${1:-}"; shift || eusage 'missing output-dir'
-suite="${1:-}"; shift || eusage 'missing suite'
-
-mkdir -p "$outputDir"
-outputDir="$(readlink -f "$outputDir")"
-
-securityArgs=(
- --cap-add SYS_ADMIN
- --cap-drop SETFCAP
-)
-if docker info | grep -q apparmor; then
- # AppArmor blocks mount :)
- securityArgs+=(
- --security-opt apparmor=unconfined
- )
-fi
-
-ver="$("$thisDir/scripts/debuerreotype-version")"
-ver="${ver%% *}"
-dockerImage="debuerreotype/debuerreotype:$ver"
-[ -z "$build" ] || docker build -t "$dockerImage" "$thisDir"
-
-ubuntuDockerImage="$dockerImage-ubuntu"
-[ -z "$build" ] || docker build -t "$ubuntuDockerImage" - <<-EODF
- FROM $dockerImage
- RUN apt-get update \\
- && apt-get install -y --no-install-recommends ubuntu-archive-keyring \\
- && rm -rf /var/lib/apt/lists/*
-EODF
-
-docker run \
- --rm \
- "${securityArgs[@]}" \
- -v /tmp \
- -w /tmp \
- -e suite="$suite" \
- -e arch="$arch" \
- -e TZ='UTC' -e LC_ALL='C' \
- "$ubuntuDockerImage" \
- bash -Eeuo pipefail -c '
- set -x
-
- dpkgArch="${arch:-$(dpkg --print-architecture | awk -F- "{ print \$NF }")}"
-
- case "$dpkgArch" in
- amd64|i386)
- mirror="http://archive.ubuntu.com/ubuntu"
- secmirror="http://security.ubuntu.com/ubuntu"
- ;;
- *)
- mirror="http://ports.ubuntu.com/ubuntu-ports"
- secmirror="$mirror" # no separate security mirror for ports
- ;;
- esac
-
- exportDir="output"
- outputDir="$exportDir/ubuntu/$dpkgArch/$suite"
-
- debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")"
-
- mkdir -p "$outputDir"
- wget -O "$outputDir/Release.gpg" "$mirror/dists/$suite/Release.gpg"
- wget -O "$outputDir/Release" "$mirror/dists/$suite/Release"
- gpgv \
- --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg \
- "$outputDir/Release.gpg" \
- "$outputDir/Release"
-
- {
- debuerreotype-init --non-debian \
- --arch="$dpkgArch" \
- --keyring /usr/share/keyrings/ubuntu-archive-keyring.gpg \
- --no-merged-usr \
- rootfs "$suite" "$mirror"
- # TODO setup proper sources.list for Ubuntu
- # deb http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse
- # deb http://archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse
- # deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse
- # deb http://security.ubuntu.com/ubuntu xenial-security main restricted universe multiverse
-
- epoch="$(< rootfs/debuerreotype-epoch)"
- touch_epoch() {
- while [ "$#" -gt 0 ]; do
- local f="$1"; shift
- touch --no-dereference --date="@$epoch" "$f"
- done
- }
-
- debuerreotype-minimizing-config rootfs
- debuerreotype-apt-get rootfs update -qq
- debuerreotype-apt-get rootfs dist-upgrade -yqq
-
- # make a couple copies of rootfs so we can create other variants
- for variant in slim sbuild; do
- mkdir "rootfs-$variant"
- tar -cC rootfs . | tar -xC "rootfs-$variant"
- done
-
- debuerreotype-apt-get rootfs install -y --no-install-recommends iproute2 iputils-ping
-
- debuerreotype-slimify rootfs-slim
-
- # this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild
- # https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26
- # https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194
- debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot
-
- create_artifacts() {
- local targetBase="$1"; shift
- local rootfs="$1"; shift
- local suite="$1"; shift
- local variant="$1"; shift
-
- if [ "$variant" != "sbuild" ]; then
- debuerreotype-tar "$rootfs" "$targetBase.tar.xz"
- else
- # sbuild needs "deb-src" entries
- debuerreotype-chroot "$rootfs" sed -ri -e "/^deb / p; s//deb-src /" /etc/apt/sources.list
-
- # APT has odd issues with "Acquire::GzipIndexes=false" + "file://..." sources sometimes
- # (which are used in sbuild for "--extra-package")
- # Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
- # ...
- # E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
- rm -f "$rootfs/etc/apt/apt.conf.d/docker-gzip-indexes"
- # TODO figure out the bug and fix it in APT instead /o\
-
- # schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar")
- # see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521
- debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz"
- fi
- du -hsx "$targetBase.tar.xz"
-
- sha256sum "$targetBase.tar.xz" | cut -d" " -f1 > "$targetBase.tar.xz.sha256"
- touch_epoch "$targetBase.tar.xz.sha256"
-
- debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest"
- echo "$epoch" > "$targetBase.debuerreotype-epoch"
- touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch"
-
- for f in debian_version os-release apt/sources.list; do
- targetFile="$targetBase.$(basename "$f" | sed -r "s/[^a-zA-Z0-9_-]+/-/g")"
- cp "$rootfs/etc/$f" "$targetFile"
- touch_epoch "$targetFile"
- done
- }
-
- for rootfs in rootfs*/; do
- rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ...
-
- du -hsx "$rootfs"
-
- variant="${rootfs#rootfs}" # "", "-slim", ...
- variant="${variant#-}" # "", "slim", ...
-
- variantDir="$outputDir/$variant"
- mkdir -p "$variantDir"
-
- targetBase="$variantDir/rootfs"
-
- create_artifacts "$targetBase" "$rootfs" "$suite" "$variant"
- done
- } >&2
-
- tar -cC "$exportDir" .
- ' | tar -xvC "$outputDir"
More information about the Neon-commits
mailing list