From f4c13e1a154debeac27da472f0713e5f9c55e79b Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Tue, 7 Nov 2023 19:00:31 +0530 Subject: [PATCH 1/8] dockerfile: reduce interval and env config - bitcoind and cln reduce healthcheck time interval - cln healthcheck no need to use switch case instead use VLS_NETWORK Signed-off-by: Lakshya Singh --- bitcoind/Dockerfile | 2 +- docker-compose.regtest.yml | 1 - docker-compose.yml | 1 - lightningd/Dockerfile | 2 +- lightningd/healthcheck.sh | 11 +---------- 5 files changed, 3 insertions(+), 14 deletions(-) diff --git a/bitcoind/Dockerfile b/bitcoind/Dockerfile index ed23efd..61d719f 100644 --- a/bitcoind/Dockerfile +++ b/bitcoind/Dockerfile @@ -133,7 +133,7 @@ ENTRYPOINT ["/entrypoint.sh"] RUN bitcoind -version | grep "Bitcoin Core version v${BITCOIN_VERSION}" -HEALTHCHECK --interval=60s --timeout=10s --start-period=30s \ +HEALTHCHECK --interval=10s --timeout=20s --start-period=30s \ CMD ["/bin/sh", "-c", "/healthcheck.sh"] CMD ["bitcoind"] diff --git a/docker-compose.regtest.yml b/docker-compose.regtest.yml index 7c4542d..afdc3c5 100644 --- a/docker-compose.regtest.yml +++ b/docker-compose.regtest.yml @@ -19,7 +19,6 @@ services: ports: - 19846:19846 environment: - - BITCOIN_CHAIN=regtest - VLS_NETWORK=regtest - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:38332 diff --git a/docker-compose.yml b/docker-compose.yml index ee3182a..e918302 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,6 @@ services: bitcoin-core: condition: service_healthy environment: - - BITCOIN_CHAIN=test - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:18332 - RUST_LOG=info - VLS_NETWORK=testnet diff --git a/lightningd/Dockerfile b/lightningd/Dockerfile index 879ca72..ed603d7 100644 --- a/lightningd/Dockerfile +++ b/lightningd/Dockerfile @@ -142,7 +142,7 @@ RUN pip3 install \ requests \ packaging -HEALTHCHECK --interval=10s --timeout=10s --start-period=15s \ +HEALTHCHECK --interval=5s --timeout=10s --start-period=5s \ CMD ["/bin/sh", "-c", "/healthcheck.sh"] ENTRYPOINT ["/entrypoint.sh"] diff --git a/lightningd/healthcheck.sh b/lightningd/healthcheck.sh index b214130..3766c35 100755 --- a/lightningd/healthcheck.sh +++ b/lightningd/healthcheck.sh @@ -2,13 +2,4 @@ set -ex -LIGHTNING_NETWORK="" - -case $BITCOIN_CHAIN in - "test") LIGHTNING_NETWORK=testnet ;; - "regtest") LIGHTNING_NETWORK=regtest ;; - "signet") LIGHTNING_NETWORK=signet ;; - *) echo "Invalid BITCOIN_CHAIN value: $BITCOIN_CHAIN" && exit 1 ;; -esac - -lightning-cli --network $LIGHTNING_NETWORK getinfo +lightning-cli --network $VLS_NETWORK summary From 6e7178a4c4739cc7826271359cbef41685d6c829 Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Tue, 7 Nov 2023 19:02:45 +0530 Subject: [PATCH 2/8] cln: remove quotes from env vars - only build remote_hsmd_socket - multiple layers for build caching - download git repo in separate command to prevent redownload - fix COPY of symlink not possible - copy the bitcoin.conf file - add git hook to ensure no difference Signed-off-by: Lakshya Singh --- hooks/pre-commit | 3 ++ lightningd/Dockerfile | 83 ++++++++++++++++++++++++----------------- lightningd/bitcoin.conf | 24 +++++++++++- 3 files changed, 75 insertions(+), 35 deletions(-) create mode 100755 hooks/pre-commit mode change 120000 => 100644 lightningd/bitcoin.conf diff --git a/hooks/pre-commit b/hooks/pre-commit new file mode 100755 index 0000000..b8bcb28 --- /dev/null +++ b/hooks/pre-commit @@ -0,0 +1,3 @@ +#!/usr/bin/sh + +cmp bitcoind/bitcoin.conf lightningd/bitcoin.conf diff --git a/lightningd/Dockerfile b/lightningd/Dockerfile index ed603d7..7a3a27b 100644 --- a/lightningd/Dockerfile +++ b/lightningd/Dockerfile @@ -1,4 +1,4 @@ -# build stage for core lightning and clboss +# build stage for core lightning and its plugins FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as builder ARG TARGETPLATFORM @@ -29,51 +29,66 @@ RUN apk update && \ ARG CORE_LIGHTNING_GIT_HASH=bd9494c65be09e6f8ecb3bc49b50a859467a65fe ARG CORE_LIGHTNING_REPO=https://github.com/lightning-signer/c-lightning.git -RUN git clone --recursive $CORE_LIGHTNING_REPO /repo && \ - cd /repo && \ +RUN git clone --recursive $CORE_LIGHTNING_REPO core-lightning +RUN cd core-lightning && \ git checkout $CORE_LIGHTNING_GIT_HASH && \ ./configure --enable-static --prefix=/usr && \ make -j $(nproc) && \ - make install + make -j $(nproc) install -RUN apk add \ - autoconf-archive \ - bind-tools \ - build-base \ - curl-dev \ - libev-dev \ - pkgconf \ - sqlite-dev +ARG PLUGINS_GIT_HASH=ce078bb74e10b5dea779fcd9fbe77e1d3e72db7a +ARG PLUGINS_REPO=https://github.com/lightningd/plugins.git +RUN git clone $PLUGINS_REPO +RUN cd plugins && \ + git checkout $PLUGINS_GIT_HASH + +FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as clboss_builder + +WORKDIR /build + +RUN apk update && \ + apk add \ + git \ + autoconf-archive \ + automake \ + bind-tools \ + build-base \ + curl-dev \ + libev-dev \ + libtool \ + pkgconf \ + sqlite-dev ARG CLBOSS_GIT_HASH=9c050d61f01ffff4b7b5d295f708569dd71b55d3 ARG CLBOSS_REPO=https://github.com/ZmnSCPxj/clboss.git -RUN git clone --recurse-submodules $CLBOSS_REPO /clboss && \ - cd /clboss && \ +RUN git clone --recurse-submodules $CLBOSS_REPO clboss +RUN cd clboss && \ git checkout $CLBOSS_GIT_HASH && \ autoreconf -i && \ ./configure --prefix=/usr && \ make -j $(nproc) && \ make install -ARG PLUGINS_GIT_HASH=ce078bb74e10b5dea779fcd9fbe77e1d3e72db7a -ARG PLUGINS_REPO=https://github.com/lightningd/plugins.git -RUN git clone $PLUGINS_REPO && \ - cd plugins && \ - git checkout $PLUGINS_GIT_HASH +FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as vls_builder -RUN apk add \ - cargo \ - rust \ - grpc \ - protobuf \ - protobuf-dev +WORKDIR /build + +RUN apk update && \ + apk add \ + build-base \ + cargo \ + git \ + grpc \ + protobuf \ + protobuf-dev \ + rust ARG VLS_GIT_HASH=af02dac8b855c361d74234387d288a9591fad1d5 ARG VLS_REPO=https://gitlab.com/lightning-signer/validating-lightning-signer.git -RUN git clone $VLS_REPO && \ - cd validating-lightning-signer && \ +RUN git clone $VLS_REPO vls +RUN cd vls && \ git checkout $VLS_GIT_HASH && \ - cargo build --bins -r + cargo build --bin remote_hsmd_socket --release # final stage with runtime dependencies and pkgs FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as runner @@ -110,15 +125,15 @@ COPY --from=builder /usr/bin/lightning-hsmtool /usr/bin/ COPY --from=builder /usr/libexec/c-lightning /usr/libexec/c-lightning COPY --from=builder /usr/share/man/man8 /usr/share/man/man8 COPY --from=builder /usr/share/doc/c-lightning /usr/share/doc/c-lightning -COPY --from=builder /usr/bin/clboss /usr/bin/clboss +COPY --from=clboss_builder /usr/bin/clboss /usr/bin/clboss COPY --from=builder /build/plugins/summary /usr/local/src/plugins/summary -COPY --from=builder /build/validating-lightning-signer/target/release/remote_hsmd_socket /usr/libexec/c-lightning/remote_hsmd_socket +COPY --from=vls_builder /build/vls/target/release/remote_hsmd_socket /usr/libexec/c-lightning/remote_hsmd_socket RUN addgroup -S lightning && adduser -S lightning -G lightning && \ mkdir -p ${LIGHTNINGD_DATA} && \ chown -R lightning:lightning ${LIGHTNINGD_DATA} -COPY bitcoin.conf "${BITCOIND_HOME}"/bitcoin.conf +COPY bitcoin.conf ${BITCOIND_HOME}/bitcoin.conf COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh @@ -128,10 +143,10 @@ RUN chmod +x /healthcheck.sh VOLUME ["${LIGHTNINGD_DATA}"] -RUN mkdir -p "${BITCOIND_HOME}" && \ - chown -R lightning:lightning "${BITCOIND_HOME}" +RUN mkdir -p ${BITCOIND_HOME} && \ + chown -R lightning:lightning ${BITCOIND_HOME} -COPY bitcoin.conf "${BITCOIND_HOME}"/bitcoin.conf +COPY bitcoin.conf ${BITCOIND_HOME}/bitcoin.conf COPY testnet-config /testnet-config COPY regtest-config /regtest-config diff --git a/lightningd/bitcoin.conf b/lightningd/bitcoin.conf deleted file mode 120000 index b1895bf..0000000 --- a/lightningd/bitcoin.conf +++ /dev/null @@ -1 +0,0 @@ -../bitcoind/bitcoin.conf \ No newline at end of file diff --git a/lightningd/bitcoin.conf b/lightningd/bitcoin.conf new file mode 100644 index 0000000..79be1bf --- /dev/null +++ b/lightningd/bitcoin.conf @@ -0,0 +1,23 @@ +## +## bitcoin.conf configuration file. Lines beginning with # are comments. +## + +server=1 +rpcallowip=0.0.0.0/0 +whitelist=0.0.0.0/0 +txindex=1 +blockfilterindex=1 + +[test] +rpcbind=0.0.0.0 +rpcuser=rpcuser +rpcpassword=VLSsigner1 +rpcport=18332 +port=18333 + +[regtest] +rpcbind=0.0.0.0 +rpcuser=rpcuser +rpcpassword=VLSsigner1 +rpcport=38332 +port=38333 From 35da6dafe0d20e5417759e70a593b933736b2ed9 Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Thu, 9 Nov 2023 22:53:00 +0530 Subject: [PATCH 3/8] vls: separate repo download and build - flexbile build option with network caching - build only vlsd2 not all - improve build speed Signed-off-by: Lakshya Singh --- vlsd/Dockerfile | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/vlsd/Dockerfile b/vlsd/Dockerfile index 7705576..f365902 100644 --- a/vlsd/Dockerfile +++ b/vlsd/Dockerfile @@ -3,22 +3,20 @@ FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as WORKDIR /build RUN apk update && \ - apk add \ - build-base \ - cargo \ - git \ - grpc \ - protobuf \ - protobuf-dev \ - rust + apk add \ + build-base \ + cargo \ + git \ + grpc \ + protobuf \ + protobuf-dev \ + rust ARG GIT_HASH=af02dac8b855c361d74234387d288a9591fad1d5 -RUN git clone https://gitlab.com/lightning-signer/validating-lightning-signer.git && \ - cd validating-lightning-signer && \ - git checkout $GIT_HASH - -RUN cd validating-lightning-signer && \ - cargo build --bins --release +RUN git clone https://gitlab.com/lightning-signer/validating-lightning-signer.git vls +RUN cd vls && \ + git checkout $GIT_HASH && \ + cargo build --bin vlsd2 --release FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as runner @@ -38,7 +36,7 @@ RUN apk update && \ protobuf \ bind-tools -COPY --from=builder /build/validating-lightning-signer/target/release/vlsd2 /usr/local/bin/vlsd2 +COPY --from=builder /build/vls/target/release/vlsd2 /usr/local/bin/vlsd2 COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh @@ -46,11 +44,11 @@ RUN chmod +x /entrypoint.sh COPY vlsd2.toml /vlsd2.toml ENV VLS_DATA=/home/vls/.lightning-signer -RUN mkdir "${VLS_DATA}" -RUN chown vls:vls "${VLS_DATA}" +RUN mkdir ${VLS_DATA} +RUN chown vls:vls ${VLS_DATA} ENV REMOTE_SIGNER_ALLOWLIST=${VLS_DATA}/ALLOWLIST -RUN touch "${REMOTE_SIGNER_ALLOWLIST}" +RUN touch ${REMOTE_SIGNER_ALLOWLIST} VOLUME ["${VLS_DATA}"] From 014042f409e92925422c41a08e168521f27fc234 Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Thu, 9 Nov 2023 22:53:42 +0530 Subject: [PATCH 4/8] txoo: separate repo download and build - add arg for txoo_data dir Signed-off-by: Lakshya Singh --- txood/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/txood/Dockerfile b/txood/Dockerfile index 9611794..5e6a18a 100644 --- a/txood/Dockerfile +++ b/txood/Dockerfile @@ -12,8 +12,8 @@ RUN apk add cargo ARG TXOO_GIT_HASH=528fa92e5def1b6cc19127f2242137bf6f13550d ARG TXOO_REPO=https://gitlab.com/lightning-signer/txoo.git -RUN git clone --recurse-submodules $TXOO_REPO && \ - cd txoo && \ +RUN git clone --recurse-submodules $TXOO_REPO +RUN cd txoo && \ git checkout $TXOO_GIT_HASH && \ cargo build --release -p txood @@ -23,6 +23,8 @@ FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as LABEL maintainer.0="Lakshya Singh (@king-11)" \ maintainer.1="Dev Random (@devrandom01)" +ARG TXOO_DATA=/root/.txoo + RUN apk update && \ apk add \ build-base \ @@ -35,7 +37,7 @@ COPY --from=builder /build/txoo/target/release/txood /usr/bin/txood COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh -VOLUME ["/root/.txoo/"] +VOLUME ["${TXOO_DATA}"] ENTRYPOINT ["/entrypoint.sh"] CMD ["txood"] From 7e5d66f55d3c83e8a2b7612bd4bdff756cc6da94 Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Mon, 20 Nov 2023 22:50:37 +0530 Subject: [PATCH 5/8] compose: change network name - not releasing binaries Signed-off-by: Lakshya Singh --- docker-compose.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e918302..6ad17b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,14 +3,14 @@ services: bitcoin-core: build: context: ./bitcoind - image: bitcoind + image: bitcoind:${IMAGE_TAG:-latest} container_name: bitcoind-test volumes: - bitcoin_data:/home/bitcoin/.bitcoin expose: - 18332 networks: - LN_testing: + lightning: aliases: - bitcoind environment: @@ -19,7 +19,7 @@ services: core-lightning: build: context: ./lightningd - image: lightningd + image: lightningd:${IMAGE_TAG:-latest} container_name: lightningd-test command: - --conf=/home/lightning/.lightning/testnet-config @@ -30,7 +30,7 @@ services: - 19735 - 7701 networks: - - LN_testing + - lightning depends_on: bitcoin-core: condition: service_healthy @@ -44,13 +44,13 @@ services: txoo: build: context: ./txood - image: txood + image: txood:${IMAGE_TAG:-latest} container_name: txood-test restart: unless-stopped command: - -r http://rpcuser:VLSsigner1@bitcoind:18332 networks: - - LN_testing + - lightning volumes: - txoo_data:/root/.txoo/ - bitcoin_data:/root/.bitcoin/ @@ -63,7 +63,7 @@ services: vls: build: context: ./vlsd - image: vlsd + image: vlsd:${IMAGE_TAG:-latest} container_name: vlsd-test profiles: - vls @@ -71,7 +71,7 @@ services: - --log-level=info - --connect=http://core-lightning:7701 networks: - - LN_testing + - lightning volumes: - vls_data:/home/vls/.lightning-signer environment: @@ -95,4 +95,4 @@ volumes: external: true networks: - LN_testing: + lightning: From eeaac24d5440ede36b13ff808bf92816c718b27f Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Thu, 23 Nov 2023 21:26:59 +0530 Subject: [PATCH 6/8] dockerfile: optimize on size and network call - merge multiple steps to reduce number of layers - run make clean to reduce layer size - run cargo clean - merge multiple ARG into single step Signed-off-by: Lakshya Singh --- bitcoind/Dockerfile | 44 ++++++++++++++++++++++--------------------- lightningd/Dockerfile | 35 +++++++++++++++++----------------- txood/Dockerfile | 10 +++++----- vlsd/Dockerfile | 5 +++-- 4 files changed, 49 insertions(+), 45 deletions(-) diff --git a/bitcoind/Dockerfile b/bitcoind/Dockerfile index 61d719f..1663d4b 100644 --- a/bitcoind/Dockerfile +++ b/bitcoind/Dockerfile @@ -53,21 +53,21 @@ ARG BITCOIN_VERSION=23.0 ARG BITCOIN_PREFIX=/opt/bitcoin-${BITCOIN_VERSION} ARG SHA256SUMS_HASH=aaff81ea001f499e8f6f3221387d7db960d71a3b7a4a2b1aaf2c8060bc94a391 -RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS -RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS.asc -RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}.tar.gz -RUN gpg --batch --verify SHA256SUMS.asc SHA256SUMS -RUN echo "${SHA256SUMS_HASH} SHA256SUMS" | sha256sum -c - -RUN grep " bitcoin-${BITCOIN_VERSION}.tar.gz\$" SHA256SUMS | sha256sum -c - -RUN tar -xzf *.tar.gz +RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS && \ + wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS.asc && \ + wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}.tar.gz && \ + gpg --batch --verify SHA256SUMS.asc SHA256SUMS && \ + echo "${SHA256SUMS_HASH} SHA256SUMS" | sha256sum -c - && \ + grep " bitcoin-${BITCOIN_VERSION}.tar.gz\$" SHA256SUMS | sha256sum -c - && \ + tar -xzf *.tar.gz WORKDIR /bitcoin-${BITCOIN_VERSION} -RUN sed -i '/AC_PREREQ/a\AR_FLAGS=cr' src/univalue/configure.ac -RUN sed -i '/AX_PROG_CC_FOR_BUILD/a\AR_FLAGS=cr' src/secp256k1/configure.ac -RUN sed -i s:sys/fcntl.h:fcntl.h: src/compat.h -RUN ./autogen.sh -RUN ./configure LDFLAGS=-L`ls -d /opt/db*`/lib/ CPPFLAGS=-I`ls -d /opt/db*`/include/ \ +RUN sed -i '/AC_PREREQ/a\AR_FLAGS=cr' src/univalue/configure.ac && \ + sed -i '/AX_PROG_CC_FOR_BUILD/a\AR_FLAGS=cr' src/secp256k1/configure.ac && \ + sed -i s:sys/fcntl.h:fcntl.h: src/compat.h && \ + ./autogen.sh && \ + ./configure LDFLAGS=-L`ls -d /opt/db*`/lib/ CPPFLAGS=-I`ls -d /opt/db*`/include/ \ --prefix=${BITCOIN_PREFIX} \ --mandir=/usr/share/man \ --disable-tests \ @@ -78,13 +78,15 @@ RUN ./configure LDFLAGS=-L`ls -d /opt/db*`/lib/ CPPFLAGS=-I`ls -d /opt/db*`/incl --with-libs \ --with-sqlite=yes \ --with-daemon -RUN make -j $(nproc) -RUN make install -RUN strip ${BITCOIN_PREFIX}/bin/bitcoin-cli -RUN strip ${BITCOIN_PREFIX}/bin/bitcoin-tx -RUN strip ${BITCOIN_PREFIX}/bin/bitcoind -RUN strip ${BITCOIN_PREFIX}/lib/libbitcoinconsensus.a -RUN strip ${BITCOIN_PREFIX}/lib/libbitcoinconsensus.so.0.0.0 + +RUN make -j $(nproc) && \ + make install && \ + strip ${BITCOIN_PREFIX}/bin/bitcoin-cli \ + ${BITCOIN_PREFIX}/bin/bitcoin-tx \ + ${BITCOIN_PREFIX}/bin/bitcoind \ + ${BITCOIN_PREFIX}/lib/libbitcoinconsensus.a \ + ${BITCOIN_PREFIX}/lib/libbitcoinconsensus.so.0.0.0 && \ + make clean # Runtime binaries and setup stage FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as runner @@ -95,8 +97,8 @@ ARG GID=101 LABEL maintainer.0="Lakshya Singh (@king-11)" \ maintainer.1="Dev Random (@devrandom01)" -RUN addgroup bitcoin --gid ${GID} --system -RUN adduser --uid ${UID} --system bitcoin --ingroup bitcoin +RUN addgroup bitcoin --gid ${GID} --system && \ + adduser --uid ${UID} --system bitcoin --ingroup bitcoin RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories RUN apk --no-cache add \ boost-filesystem \ diff --git a/lightningd/Dockerfile b/lightningd/Dockerfile index 7a3a27b..9de94cf 100644 --- a/lightningd/Dockerfile +++ b/lightningd/Dockerfile @@ -27,17 +27,18 @@ RUN apk update && \ zlib-dev \ zlib-static -ARG CORE_LIGHTNING_GIT_HASH=bd9494c65be09e6f8ecb3bc49b50a859467a65fe -ARG CORE_LIGHTNING_REPO=https://github.com/lightning-signer/c-lightning.git +ARG CORE_LIGHTNING_GIT_HASH=bd9494c65be09e6f8ecb3bc49b50a859467a65fe \ + CORE_LIGHTNING_REPO=https://github.com/lightning-signer/c-lightning.git RUN git clone --recursive $CORE_LIGHTNING_REPO core-lightning RUN cd core-lightning && \ git checkout $CORE_LIGHTNING_GIT_HASH && \ ./configure --enable-static --prefix=/usr && \ make -j $(nproc) && \ - make -j $(nproc) install + make -j $(nproc) install && \ + make clean -ARG PLUGINS_GIT_HASH=ce078bb74e10b5dea779fcd9fbe77e1d3e72db7a -ARG PLUGINS_REPO=https://github.com/lightningd/plugins.git +ARG PLUGINS_GIT_HASH=ce078bb74e10b5dea779fcd9fbe77e1d3e72db7a \ + PLUGINS_REPO=https://github.com/lightningd/plugins.git RUN git clone $PLUGINS_REPO RUN cd plugins && \ git checkout $PLUGINS_GIT_HASH @@ -59,15 +60,16 @@ RUN apk update && \ pkgconf \ sqlite-dev -ARG CLBOSS_GIT_HASH=9c050d61f01ffff4b7b5d295f708569dd71b55d3 -ARG CLBOSS_REPO=https://github.com/ZmnSCPxj/clboss.git +ARG CLBOSS_GIT_HASH=9c050d61f01ffff4b7b5d295f708569dd71b55d3 \ + CLBOSS_REPO=https://github.com/ZmnSCPxj/clboss.git RUN git clone --recurse-submodules $CLBOSS_REPO clboss RUN cd clboss && \ git checkout $CLBOSS_GIT_HASH && \ autoreconf -i && \ ./configure --prefix=/usr && \ make -j $(nproc) && \ - make install + make -j $(nproc) install && \ + make clean FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as vls_builder @@ -83,12 +85,13 @@ RUN apk update && \ protobuf-dev \ rust -ARG VLS_GIT_HASH=af02dac8b855c361d74234387d288a9591fad1d5 -ARG VLS_REPO=https://gitlab.com/lightning-signer/validating-lightning-signer.git +ARG VLS_GIT_HASH=af02dac8b855c361d74234387d288a9591fad1d5 \ + VLS_REPO=https://gitlab.com/lightning-signer/validating-lightning-signer.git RUN git clone $VLS_REPO vls RUN cd vls && \ git checkout $VLS_GIT_HASH && \ - cargo build --bin remote_hsmd_socket --release + cargo install --path ./vls-proxy --bin remote_hsmd_socket --profile release --root /usr/local/ && \ + cargo clean # final stage with runtime dependencies and pkgs FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as runner @@ -109,8 +112,8 @@ RUN apk update && \ curl-dev \ sqlite-dev -ARG LIGHTNINGD_UID=101 -ARG LIGHTNINGD_USER=lightning +ARG LIGHTNINGD_UID=101 \ + LIGHTNINGD_USER=lightning ARG LIGHTNINGD_HOME=/home/${LIGHTNINGD_USER} ENV LIGHTNINGD_DATA=${LIGHTNINGD_HOME}/.lightning \ LIGHTNINGD_RPC_PORT=9835 \ @@ -119,15 +122,13 @@ ENV LIGHTNINGD_DATA=${LIGHTNINGD_HOME}/.lightning \ RUN mkdir -p /usr/local/src/plugins -COPY --from=builder /usr/bin/lightningd /usr/bin/ -COPY --from=builder /usr/bin/lightning-cli /usr/bin/ -COPY --from=builder /usr/bin/lightning-hsmtool /usr/bin/ +COPY --from=builder /usr/bin/lightningd /usr/bin/lightning-cli /usr/bin/lightning-hsmtool /usr/bin/ COPY --from=builder /usr/libexec/c-lightning /usr/libexec/c-lightning COPY --from=builder /usr/share/man/man8 /usr/share/man/man8 COPY --from=builder /usr/share/doc/c-lightning /usr/share/doc/c-lightning COPY --from=clboss_builder /usr/bin/clboss /usr/bin/clboss COPY --from=builder /build/plugins/summary /usr/local/src/plugins/summary -COPY --from=vls_builder /build/vls/target/release/remote_hsmd_socket /usr/libexec/c-lightning/remote_hsmd_socket +COPY --from=vls_builder /usr/local/bin/remote_hsmd_socket /usr/libexec/c-lightning/remote_hsmd_socket RUN addgroup -S lightning && adduser -S lightning -G lightning && \ mkdir -p ${LIGHTNINGD_DATA} && \ diff --git a/txood/Dockerfile b/txood/Dockerfile index 5e6a18a..a8fdd52 100644 --- a/txood/Dockerfile +++ b/txood/Dockerfile @@ -6,16 +6,16 @@ WORKDIR /build RUN apk update && \ apk add \ git \ - rust - -RUN apk add cargo + rust \ + cargo ARG TXOO_GIT_HASH=528fa92e5def1b6cc19127f2242137bf6f13550d ARG TXOO_REPO=https://gitlab.com/lightning-signer/txoo.git RUN git clone --recurse-submodules $TXOO_REPO RUN cd txoo && \ git checkout $TXOO_GIT_HASH && \ - cargo build --release -p txood + cargo install --path ./txood --bin txood --profile release --root /usr/local/ && \ + cargo clean # txoo runner FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as runner @@ -32,7 +32,7 @@ RUN apk update && \ libev-dev \ curl-dev -COPY --from=builder /build/txoo/target/release/txood /usr/bin/txood +COPY --from=builder /usr/local/bin/txood /usr/bin/txood COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/vlsd/Dockerfile b/vlsd/Dockerfile index f365902..e1c681a 100644 --- a/vlsd/Dockerfile +++ b/vlsd/Dockerfile @@ -16,7 +16,8 @@ ARG GIT_HASH=af02dac8b855c361d74234387d288a9591fad1d5 RUN git clone https://gitlab.com/lightning-signer/validating-lightning-signer.git vls RUN cd vls && \ git checkout $GIT_HASH && \ - cargo build --bin vlsd2 --release + cargo install --path ./vls-proxy --bin vlsd2 --profile release --root /usr/local/ && \ + cargo clean FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as runner @@ -36,7 +37,7 @@ RUN apk update && \ protobuf \ bind-tools -COPY --from=builder /build/vls/target/release/vlsd2 /usr/local/bin/vlsd2 +COPY --from=builder /usr/local/bin/vlsd2 /usr/local/bin/vlsd2 COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh From e24d968f27c39597047c3466086bace8b4093a10 Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Fri, 1 Dec 2023 02:54:42 +0530 Subject: [PATCH 7/8] dockerfile: update vls & txoo git hash - fix for latest rust vls - txoo fix block_iterator update Signed-off-by: Lakshya Singh --- lightningd/Dockerfile | 2 +- txood/Dockerfile | 2 +- vlsd/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lightningd/Dockerfile b/lightningd/Dockerfile index 9de94cf..464ef33 100644 --- a/lightningd/Dockerfile +++ b/lightningd/Dockerfile @@ -85,7 +85,7 @@ RUN apk update && \ protobuf-dev \ rust -ARG VLS_GIT_HASH=af02dac8b855c361d74234387d288a9591fad1d5 \ +ARG VLS_GIT_HASH=59544fd5b15658643acebdaf18886022db3d1071 \ VLS_REPO=https://gitlab.com/lightning-signer/validating-lightning-signer.git RUN git clone $VLS_REPO vls RUN cd vls && \ diff --git a/txood/Dockerfile b/txood/Dockerfile index a8fdd52..bfa8b13 100644 --- a/txood/Dockerfile +++ b/txood/Dockerfile @@ -9,7 +9,7 @@ RUN apk update && \ rust \ cargo -ARG TXOO_GIT_HASH=528fa92e5def1b6cc19127f2242137bf6f13550d +ARG TXOO_GIT_HASH=1cfeecf5f806a04a12c97ca17c39566a1a24e732 ARG TXOO_REPO=https://gitlab.com/lightning-signer/txoo.git RUN git clone --recurse-submodules $TXOO_REPO RUN cd txoo && \ diff --git a/vlsd/Dockerfile b/vlsd/Dockerfile index e1c681a..edd7e4a 100644 --- a/vlsd/Dockerfile +++ b/vlsd/Dockerfile @@ -12,7 +12,7 @@ RUN apk update && \ protobuf-dev \ rust -ARG GIT_HASH=af02dac8b855c361d74234387d288a9591fad1d5 +ARG GIT_HASH=59544fd5b15658643acebdaf18886022db3d1071 RUN git clone https://gitlab.com/lightning-signer/validating-lightning-signer.git vls RUN cd vls && \ git checkout $GIT_HASH && \ From e926f4aaccd7dad8e8154211a3b5aed6517fe718 Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Tue, 7 Nov 2023 19:15:20 +0530 Subject: [PATCH 8/8] ci: add gitlab ci for docker compose - split into multiple jobs - specify tags for runner large - use buildx and add separate builder default has no cache - build image from cache and push to registry - compose use tag and pull - use cache tag for reuse - https://github.com/moby/buildkit/issues/1981 - https://github.com/moby/buildkit/issues/3730 - use cache-to and cache-from with registry/cache - add dockerignore to prevent rebuild on git changes - use individual builders - add step for docker compose up - update readme - update dockerignore - add gitignore - use script reduce duplicate code - cache from and to in build, compose use cache - dont push images just use cache - build in compose step from cache Signed-off-by: Lakshya Singh --- .dockerignore | 9 ++++++ .gitignore | 1 + .gitlab-ci.yml | 57 +++++++++++++++++++++++++++++++++++ README.md | 2 +- scripts/build_from_cache.sh | 8 +++++ scripts/build_image_cache.sh | 10 ++++++ {hooks => scripts}/pre-commit | 0 7 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100755 scripts/build_from_cache.sh create mode 100755 scripts/build_image_cache.sh rename {hooks => scripts}/pre-commit (100%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fed5c56 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/docker-compose* +**/Dockerfile* +**/*.Dockerfile +LICENSE +README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c49bd7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..8a8808f --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,57 @@ +stages: + - build + - test + +default: + image: docker:24.0.7 + services: + - docker:24.0.7-dind + before_script: + - docker version + - docker compose version + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + +variables: + VLS_CONTAINER_REGISTRY: $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME + CACHE: $CI_REGISTRY_IMAGE/cache + CACHE_TAG: $CI_COMMIT_REF_SLUG + IMAGE_TAG: $CI_COMMIT_SHA + +build_bitcoin: + tags: [ saas-linux-large-amd64 ] + stage: build + script: + - ./scripts/build_image_cache.sh bitcoind +build_core_lightning: + tags: [ saas-linux-large-amd64 ] + stage: build + script: + - ./scripts/build_image_cache.sh lightningd +build_txoo: + tags: [ saas-linux-medium-amd64 ] + stage: build + script: + - ./scripts/build_image_cache.sh txood +build_vls: + tags: [ saas-linux-large-amd64 ] + stage: build + script: + - ./scripts/build_image_cache.sh vlsd +compose: + tags: [ saas-linux-medium-amd64 ] + stage: test + variables: + BUILDER: compose_builder + before_script: + - docker volume create bitcoin_data + - docker volume create lightning_data + - docker volume create txoo_data + - docker volume create vls_data + - docker buildx create --name=$BUILDER --use + script: + - echo "IMAGE_TAG=${IMAGE_TAG}" > .env + - ./scripts/build_from_cache.sh bitcoind + - ./scripts/build_from_cache.sh lightningd + - ./scripts/build_from_cache.sh txood + - ./scripts/build_from_cache.sh vlsd + - docker compose --profile vls up -d --wait --wait-timeout 120 --build diff --git a/README.md b/README.md index 7bea844..fcede27 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ docker container exec lightningd-test lightning-cli --testnet summary - [x] standalone docker image for `lightningd` - [x] standalone docker image for `bitcoind` - [ ] supporting signet -- [ ] Profile configuration to run `vls` +- [x] Profile configuration to run `vls` - [ ] Healthcheck for `txoo` - [ ] Healthcheck for `vls` - [ ] Lightning Storage Server Dockerfile and Compose Service diff --git a/scripts/build_from_cache.sh b/scripts/build_from_cache.sh new file mode 100755 index 0000000..2f50344 --- /dev/null +++ b/scripts/build_from_cache.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -ex + +SERVICE=$1 + +cd $SERVICE +docker build --load --builder $BUILDER --cache-from $CACHE/$SERVICE:$CACHE_TAG -t $SERVICE:$IMAGE_TAG . diff --git a/scripts/build_image_cache.sh b/scripts/build_image_cache.sh new file mode 100755 index 0000000..bbab802 --- /dev/null +++ b/scripts/build_image_cache.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -ex + +SERVICE=$1 + +docker buildx create --name=$SERVICE --use +cd $SERVICE + +docker build --load --builder=$SERVICE --cache-to type=registry,ref=$CACHE/$SERVICE:$CACHE_TAG --cache-from $CACHE/$SERVICE:$CACHE_TAG -t $SERVICE:$IMAGE_TAG . diff --git a/hooks/pre-commit b/scripts/pre-commit similarity index 100% rename from hooks/pre-commit rename to scripts/pre-commit