diff --git a/.dockerignore b/.dockerignore index 84ee65c..3b7aae8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,9 @@ -**/.dockerignore -**/.env -**/.git -**/.gitignore +.git +.gitignore **/docker-compose* **/Dockerfile* -**/*.Dockerfile +.dockerignore LICENSE README.md -scripts/ +scripts .gitlab-ci.yml diff --git a/.env b/.env new file mode 100644 index 0000000..ffaba2b --- /dev/null +++ b/.env @@ -0,0 +1,18 @@ +# bitcoin version 26.0 +BITCOIN_VERSION=26.0 +BITCOIN_SHA256SUMS_HASH=63487c308a6655f939efd700bfca8e0285fa2f869283aaa7192bdd4b8799a747 +# core lightning version v24.02 +CORE_LIGHTNING_REPO=https://github.com/ElementsProject/lightning.git +CORE_LIGHTNING_GIT_HASH=8418989f9bf3563a67ac91602500672b51628579 +# clboss +CLBOSS_REPO=https://github.com/ZmnSCPxj/clboss.git +CLBOSS_GIT_HASH=0673c50e7374ea8f5cb7e302f72b7978c6bd1794 +# cln plugins +CLN_PLUGINS_REPO=https://github.com/lightningd/plugins.git +CLN_PLUGINS_GIT_HASH=a525e6c42033a6270c81065a559301ceb2a761fa +# txoo version 0.6.4 +TXOO_REPO=https://gitlab.com/lightning-signer/txoo.git +TXOO_GIT_HASH=6f0718e3f2b9406df5e3cd73306f473199141da0 +# vls version v0.11.0 +VLS_REPO=https://gitlab.com/lightning-signer/validating-lightning-signer.git +VLS_GIT_HASH=395b604964d5a0b4e9ee32c1b6e440dfbf4874f0 diff --git a/.gitignore b/.gitignore index 4c49bd7..e69de29 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +0,0 @@ -.env diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b845123..4ff38e1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,7 +49,7 @@ compose: - docker volume create vls_data - docker buildx create --name=$BUILDER --use script: - - echo "IMAGE_TAG=${IMAGE_TAG}" > .env + - export IMAGE_TAG=${IMAGE_TAG} - ./scripts/build_from_cache.sh bitcoind - ./scripts/build_from_cache.sh lightningd - ./scripts/build_from_cache.sh txood diff --git a/README.md b/README.md index 0da5147..1fc962a 100644 --- a/README.md +++ b/README.md @@ -71,10 +71,11 @@ There is a `docker-compose.yml` in the `vlsd` folder which can be used to run a ```bash cd vlsd +export VLS_GIT_HASH=$VLS_GIT_HASH docker compose up ``` -**_Note_**: Make sure to set `BITCOIND_RPC_URL` and `CLN_RMEOTE_HSMD_URL` as either environment variables or in the `docker-compose.yml` file before running the above command. +**_Note_**: Make sure to set `BITCOIND_RPC_URL`, `VLS_GIT_HASH` and `CLN_RMEOTE_HSMD_URL` as either environment variables or in the `docker-compose.yml` file before running the above command. You can take `VLS_GIT_HASH` from [.env](./.env) or from the main [repo](https://gitlab.com/lightning-signer/validating-lightning-signer) If you wish to run it as a standalone container without using `docker-compose` you can use the following command: @@ -84,6 +85,7 @@ docker run \ --rm \ --name vlsd \ --network host \ + --build_arg VLS_GIT_HASH=$VLS_GIT_HASH -e VLS_NETWORK=testnet \ -e BITCOIND_RPC_URL=$BITCOIND_RPC_URL \ --mount 'type=volume,src=vls_data,dst=/home/vls/.lightning-signer' \ @@ -141,16 +143,11 @@ docker compose up --build Above command will run `bitcoind`, `lightningd` and `txood` services on a single node. -### Additional Regtest Commands +## Interacting with Containers -We have to run these commands after bitcoind is up and running. +We can use the `docker container exec ` command to interact with containers and interact with setup using command line tools like `bitcoin-cli`, `lightning-cli`, `vls-cli`, etc. -Create wallet and make it load automatically: -```bash -docker container exec bitcoind-regtest bitcoin-cli createwallet default -docker container exec bitcoind-regtest bitcoin-cli unloadwallet default -docker container exec bitcoind-regtest bitcoin-cli loadwallet default true -``` +### Regtest Commands Generate Address for node: ```bash @@ -182,6 +179,52 @@ docker container exec lightningd-test lightning-cli --testnet clboss-status | le docker container exec lightningd-test lightning-cli --testnet summary ``` +## Debugging Containers + +Checking running status and health of containers +```bash +docker ps +``` + +Getting logs from container +```bash +# CONTAINER_NAME=lightningd-test +docker container logs $CONTAINER_NAME +``` + +Restarting containers +```bash +cd vls-container +docker compose --profile vls stop +# to start testnet +docker compose --profile vls up +``` + +Delete containers +```bash +cd vls-container +docker compose --profile vls down +``` + +## Choosing Versions + +The currently set default versions for services is as follows in the [.env](.env) file: +- **Bitcoin Core**: v26.0 +- **Core Lightning**: v24.02 +- **TXOO**: v0.6.4 +- **VLS**: v0.11.0 + +You just can switch to a particular version/commit for a service by updating the git hash and then rebuilding the service: +```bash +cd vls-container +# update bitcoin core version +sed -i 's/23.0/23.2/g' .env +# build images again +docker compose --profile vls build +``` + +Note: For `bitcoind` its also important to update the `BITCOIN_SHA256SUMS_HASH`. It is the *SHA256 HASH* of `SHA256SUMS` file. + ## Future Work - [x] standalone dockerfile for vlsd diff --git a/bitcoind/Dockerfile b/bitcoind/Dockerfile index 69cd2af..00c9a10 100644 --- a/bitcoind/Dockerfile +++ b/bitcoind/Dockerfile @@ -25,47 +25,44 @@ RUN mkdir $GNUPGHOME && set -ex \ 152812300785C96444D3334D17565732E08E5E41 \ 0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 \ 590B7292695AFFA5B672CBB2E13FC145CD3F4304 \ - 28F5900B1BB5D1A4B6B6D1A9ED357015286A333D \ - 637DB1E23370F84AFF88CCE03152347D07DA627C \ CFB16E21C950F67FA95E558F2EEB9F5CC09526C1 \ F4FC70F07310028424EFC20A8E4256593F177720 \ D1DBF2C4B96F2DEBF4C16654410108112E7EA81F \ 287AE4CA1187C68C08B49CB2D11BD4F33F1DB499 \ - F9A8737BF4FF5C89C903DF31DD78544CF91B1514 \ 9DEAE0DC7063249FB05474681E4AED62986CD25D \ - E463A93F5F3117EEDE6C7316BD02942421F4889F \ - 9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C \ - 4DAF18FE948E7A965B30F9457E296D555E7F63A7 \ + 3EB0DEE6004A13BE5A0CC758BF2978B068054311 \ + ED9BDF7AD6A55E232E84524257FF9BDBCC301009 \ 28E72909F1717FE9607754F8A7BEB2621678D37D \ - 74E2DEF5D77260B98BC19438099BAD163C70FBFA \ ; do \ gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" || \ gpg --batch --keyserver keys.openpgp.org --recv-keys "$key" || \ + gpg --batch --keyserver pgp.mit.edu --recv-keys "$key" || \ gpg --batch --keyserver keyserver.pgp.com --recv-keys "$key" || \ gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" || \ gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" ; \ - done && \ - wget -O- https://raw.githubusercontent.com/Kvaciral/kvaciral/main/kvaciral.asc | gpg --import - + done +RUN gpg --keyserver hkps://keys.openpgp.org --refresh-keys RUN gpg -kv -ARG BITCOIN_VERSION=23.0 -ARG BITCOIN_PREFIX=/opt/bitcoin-${BITCOIN_VERSION} -ARG SHA256SUMS_HASH=aaff81ea001f499e8f6f3221387d7db960d71a3b7a4a2b1aaf2c8060bc94a391 +COPY verify.sh verify.sh +ARG BITCOIN_VERSION +ARG BITCOIN_PREFIX=/opt/bitcoin-${BITCOIN_VERSION} +ARG BITCOIN_SHA256SUMS_HASH +RUN echo "Building bitcoin core version $BITCOIN_VERSION" RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS && \ wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS.asc && \ wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}.tar.gz && \ - gpg --batch --verify SHA256SUMS.asc SHA256SUMS && \ - echo "${SHA256SUMS_HASH} SHA256SUMS" | sha256sum -c - && \ + ./verify.sh && \ + echo "${BITCOIN_SHA256SUMS_HASH} SHA256SUMS" | sha256sum -c - && \ grep " bitcoin-${BITCOIN_VERSION}.tar.gz\$" SHA256SUMS | sha256sum -c - && \ tar -xzf *.tar.gz WORKDIR /bitcoin-${BITCOIN_VERSION} -RUN sed -i '/AC_PREREQ/a\AR_FLAGS=cr' src/univalue/configure.ac && \ +RUN sed -i '/AC_PREREQ/a\AR_FLAGS=cr' configure.ac && \ sed -i '/AX_PROG_CC_FOR_BUILD/a\AR_FLAGS=cr' src/secp256k1/configure.ac && \ - sed -i s:sys/fcntl.h:fcntl.h: src/compat.h && \ + sed -i s:sys/fcntl.h:fcntl.h: src/compat/compat.h && \ ./autogen.sh && \ ./configure LDFLAGS=-L`ls -d /opt/db*`/lib/ CPPFLAGS=-I`ls -d /opt/db*`/include/ \ --prefix=${BITCOIN_PREFIX} \ @@ -108,10 +105,11 @@ RUN apk --no-cache add \ libevent \ libzmq -ENV BITCOIN_DATA=/home/bitcoin/.bitcoin -ARG BITCOIN_VERSION=23.0 +ARG BITCOIN_VERSION ARG BITCOIN_PREFIX=/opt/bitcoin-${BITCOIN_VERSION} +ENV BITCOIN_DATA=/home/bitcoin/.bitcoin + COPY --from=bitcoin-core ${BITCOIN_PREFIX}/bin/bitcoin-cli /usr/bin/bitcoin-cli COPY --from=bitcoin-core ${BITCOIN_PREFIX}/bin/bitcoin-tx /usr/bin/bitcoin-tx COPY --from=bitcoin-core ${BITCOIN_PREFIX}/bin/bitcoind /usr/bin/bitcoind diff --git a/bitcoind/verify.sh b/bitcoind/verify.sh new file mode 100755 index 0000000..23df4aa --- /dev/null +++ b/bitcoind/verify.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +# verify SHA256SUMS +gpg --batch --verify SHA256SUMS.asc SHA256SUMS +ret_val=$? + +# allow 2 as well in case of untrusted keys +if [ $ret_val -eq 0 ] || [ $ret_val -eq 2 ]; then + exit 0 +else + exit $ret_val +fi \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 368ba2a..5fe055d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,9 @@ services: bitcoin-core: build: context: ./bitcoind + args: + - BITCOIN_VERSION + - BITCOIN_SHA256SUMS_HASH image: bitcoind:${IMAGE_TAG:-latest} container_name: bitcoind-test volumes: @@ -19,6 +22,15 @@ services: core-lightning: build: context: ./lightningd + args: + - CORE_LIGHTNING_REPO + - CORE_LIGHTNING_GIT_HASH + - CLBOSS_REPO + - CLBOSS_GIT_HASH + - CLN_PLUGINS_REPO + - CLN_PLUGINS_GIT_HASH + - VLS_REPO + - VLS_GIT_HASH image: lightningd:${IMAGE_TAG:-latest} container_name: lightningd-test command: @@ -46,6 +58,9 @@ services: txoo: build: context: ./txood + args: + - TXOO_REPO + - TXOO_GIT_HASH image: txood:${IMAGE_TAG:-latest} container_name: txood-test restart: unless-stopped @@ -64,6 +79,9 @@ services: vls: build: context: ./vlsd + args: + - VLS_REPO + - VLS_GIT_HASH image: vlsd:${IMAGE_TAG:-latest} container_name: vlsd-test profiles: diff --git a/lightningd/Dockerfile b/lightningd/Dockerfile index 1650543..fee2ef5 100644 --- a/lightningd/Dockerfile +++ b/lightningd/Dockerfile @@ -20,6 +20,7 @@ RUN apk update && \ net-tools \ postgresql-dev \ py3-mako \ + py3-pip \ python3 \ python3-dev \ sqlite-dev \ @@ -27,21 +28,22 @@ RUN apk update && \ zlib-dev \ zlib-static -ARG CORE_LIGHTNING_GIT_HASH=76cd32387a400a2aee607edaf1e0a06e92e1402f \ - CORE_LIGHTNING_REPO=https://github.com/ElementsProject/lightning.git +RUN pip3 install grpcio-tools + +ARG CORE_LIGHTNING_GIT_HASH \ + CORE_LIGHTNING_REPO RUN git clone --recursive $CORE_LIGHTNING_REPO core-lightning RUN cd core-lightning && \ git checkout $CORE_LIGHTNING_GIT_HASH && \ ./configure --enable-static --prefix=/usr && \ - make -j $(nproc) && \ make -j $(nproc) install && \ make clean -ARG PLUGINS_GIT_HASH=cf96eb63c1687644042ea85acfd3785527fd98d4 \ - PLUGINS_REPO=https://github.com/lightningd/plugins.git -RUN git clone $PLUGINS_REPO +ARG CLN_PLUGINS_GIT_HASH \ + CLN_PLUGINS_REPO +RUN git clone $CLN_PLUGINS_REPO RUN cd plugins && \ - git checkout $PLUGINS_GIT_HASH + git checkout $CLN_PLUGINS_GIT_HASH FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as clboss_builder @@ -60,8 +62,8 @@ RUN apk update && \ pkgconf \ sqlite-dev -ARG CLBOSS_GIT_HASH=0673c50e7374ea8f5cb7e302f72b7978c6bd1794 \ - CLBOSS_REPO=https://github.com/ZmnSCPxj/clboss.git +ARG CLBOSS_GIT_HASH \ + CLBOSS_REPO RUN git clone --recurse-submodules $CLBOSS_REPO clboss RUN cd clboss && \ git checkout $CLBOSS_GIT_HASH && \ @@ -85,9 +87,9 @@ RUN apk update && \ protobuf-dev \ rust -ARG VLS_GIT_HASH=5712eb74b672ffd8b691b9e772e05a827d03e7df \ - VLS_REPO=https://gitlab.com/lightning-signer/validating-lightning-signer.git -RUN echo building remote_hsmd_socket from hash: $VLS_GIT_HASH +ARG VLS_GIT_HASH \ + VLS_REPO +RUN echo "building remote_hsmd_socket from hash: $VLS_GIT_HASH" RUN git clone $VLS_REPO vls RUN cd vls && \ git checkout $VLS_GIT_HASH && \ @@ -128,7 +130,7 @@ COPY --from=builder /usr/libexec/c-lightning /usr/libexec/c-lightning COPY --from=builder /usr/share/man/man8 /usr/share/man/man8 COPY --from=builder /usr/share/doc/c-lightning /usr/share/doc/c-lightning COPY --from=clboss_builder /usr/bin/clboss /usr/bin/clboss -COPY --from=builder /build/plugins/summary /usr/local/src/plugins/summary +COPY --from=builder /build/plugins/archived/summary /usr/local/src/plugins/summary COPY --from=vls_builder /usr/local/bin/remote_hsmd_socket /usr/libexec/c-lightning/remote_hsmd_socket RUN addgroup -S lightning && adduser -S lightning -G lightning && \ diff --git a/scripts/build_from_cache.sh b/scripts/build_from_cache.sh index 2f50344..458b0d0 100755 --- a/scripts/build_from_cache.sh +++ b/scripts/build_from_cache.sh @@ -5,4 +5,5 @@ set -ex SERVICE=$1 cd $SERVICE -docker build --load --builder $BUILDER --cache-from $CACHE/$SERVICE:$CACHE_TAG -t $SERVICE:$IMAGE_TAG . + +docker build --load --builder $BUILDER --cache-from $CACHE/$SERVICE:$CACHE_TAG -t $SERVICE:$IMAGE_TAG $(grep -v '^#' ../.env | sed 's/^/--build-arg /' | tr '\n' ' ') . diff --git a/scripts/build_image_cache.sh b/scripts/build_image_cache.sh index bbab802..8934c5f 100755 --- a/scripts/build_image_cache.sh +++ b/scripts/build_image_cache.sh @@ -7,4 +7,4 @@ SERVICE=$1 docker buildx create --name=$SERVICE --use cd $SERVICE -docker build --load --builder=$SERVICE --cache-to type=registry,ref=$CACHE/$SERVICE:$CACHE_TAG --cache-from $CACHE/$SERVICE:$CACHE_TAG -t $SERVICE:$IMAGE_TAG . +docker build --load --builder=$SERVICE --cache-to type=registry,ref=$CACHE/$SERVICE:$CACHE_TAG --cache-from $CACHE/$SERVICE:$CACHE_TAG -t $SERVICE:$IMAGE_TAG $(grep -v '^#' ../.env | sed 's/^/--build-arg /' | tr '\n' ' ') . diff --git a/txood/Dockerfile b/txood/Dockerfile index 0a51055..66a10fd 100644 --- a/txood/Dockerfile +++ b/txood/Dockerfile @@ -9,8 +9,8 @@ RUN apk update && \ rust \ cargo -ARG TXOO_GIT_HASH=1cfeecf5f806a04a12c97ca17c39566a1a24e732 -ARG TXOO_REPO=https://gitlab.com/lightning-signer/txoo.git +ARG TXOO_GIT_HASH \ + TXOO_REPO RUN git clone --recurse-submodules $TXOO_REPO RUN cd txoo && \ git checkout $TXOO_GIT_HASH && \ diff --git a/vlsd/Dockerfile b/vlsd/Dockerfile index 10e77b6..809a9aa 100644 --- a/vlsd/Dockerfile +++ b/vlsd/Dockerfile @@ -12,9 +12,10 @@ RUN apk update && \ protobuf-dev \ rust -ARG VLS_GIT_HASH=5712eb74b672ffd8b691b9e772e05a827d03e7df -RUN echo building vlsd from hash: $VLS_GIT_HASH -RUN git clone https://gitlab.com/lightning-signer/validating-lightning-signer.git vls +ARG VLS_GIT_HASH \ + VLS_REPO +RUN echo "building vlsd from hash: $VLS_GIT_HASH" +RUN git clone $VLS_REPO vls RUN cd vls && \ git checkout $VLS_GIT_HASH && \ cargo install --locked --path ./vls-proxy --bin vlsd2 --profile release --root /usr/local/ && \ diff --git a/vlsd/docker-compose.yml b/vlsd/docker-compose.yml index ac067c3..1bba45e 100644 --- a/vlsd/docker-compose.yml +++ b/vlsd/docker-compose.yml @@ -4,6 +4,8 @@ services: build: context: . dockerfile: Dockerfile + args: + - VLS_GIT_HASH image: vlsd container_name: vlsd-standalone command: