Merge branch 'king-11/update-vls' into 'main'

VLS: version v0.12.0

See merge request lightning-signer/vls-container!30

.env

@@ -4,18 +4,18 @@ BITCOIN_SHA256SUMS_HASH=63487c308a6655f939efd700bfca8e0285fa2f869283aaa7192bdd4b
 # core lightning version v24.05
 CORE_LIGHTNING_REPO=https://github.com/ElementsProject/lightning.git
 CORE_LIGHTNING_GIT_HASH=11586abf79cad33727c03dfa810ae8bcdd3762bf
-# clboss version v0.13.1 (2024-05-31 21:28:55 -0700)
+# clboss version v0.14.0-rc1
-CLBOSS_REPO=https://github.com/ZmnSCPxj/clboss.git
+CLBOSS_REPO=https://github.com/vincenzopalazzo/clboss
-CLBOSS_GIT_HASH=df51d5486b05acbeca16cb9e42d18fa082e6079a
+CLBOSS_GIT_HASH=37d493cdf8360c83621d20ac54543b045ae0ef21
-# cln plugins (2024-06-06 17:02:52 +0200)
+# cln plugins (2024-07-27)
 CLN_PLUGINS_REPO=https://github.com/lightningd/plugins.git
-CLN_PLUGINS_GIT_HASH=de0c7af343df25e82536233ac248ef46efea8670
+CLN_PLUGINS_GIT_HASH=5e449468bd57db7d0f33178fe0dc867e0da94133
-# txoo version 0.6.4
+# txoo version 0.8.1
 TXOO_REPO=https://gitlab.com/lightning-signer/txoo.git
-TXOO_GIT_HASH=6f0718e3f2b9406df5e3cd73306f473199141da0
+TXOO_GIT_HASH=d2f698c1434a91d5d53be4fd96ff01d3c5dfa22a
-# vls version v0.11.0 (2024-06-06 15:09:25 +0000)
+# vls version v0.12.0
 VLS_REPO=https://gitlab.com/lightning-signer/validating-lightning-signer.git
-VLS_GIT_HASH=d2590ba34a388a016bb91307864e993fb3cc3d84
+VLS_GIT_HASH=c17067773b0bc57b27a6a0c7127c48867563e280
 # lss version v0.1.0
 LSS_REPO=https://gitlab.com/lightning-signer/validating-lightning-signer.git
 LSS_GIT_HASH=d2590ba34a388a016bb91307864e993fb3cc3d84

.gitlab-ci.yml

@@ -3,13 +3,13 @@ stages:
   - test
 
 default:
-  image: docker:24.0.7
+  image: docker:26.0.1
   services:
-    - docker:24.0.7-dind
+    - docker:26.0.1-dind
   before_script:
-  - docker version
+    - docker version
-  - docker compose version
+    - docker compose version
-  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
 variables:
   VLS_CONTAINER_REGISTRY: $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME
@@ -18,17 +18,17 @@ variables:
   IMAGE_TAG: $CI_COMMIT_SHA
 
 build_bitcoin:
-  tags: [ saas-linux-large-amd64 ]
+  tags: [saas-linux-large-amd64]
   stage: build
   script:
     - ./scripts/build_image_cache.sh bitcoind
 build_core_lightning:
-  tags: [ saas-linux-large-amd64 ]
+  tags: [saas-linux-large-amd64]
   stage: build
   script:
     - ./scripts/build_image_cache.sh lightningd
 build_txoo:
-  tags: [ saas-linux-medium-amd64 ]
+  tags: [saas-linux-medium-amd64]
   stage: build
   script:
     - ./scripts/build_image_cache.sh txood
@@ -38,20 +38,16 @@ build_lss:
   script:
     - ./scripts/build_image_cache.sh lss
 build_vls:
-  tags: [ saas-linux-large-amd64 ]
+  tags: [saas-linux-large-amd64]
   stage: build
   script:
     - ./scripts/build_image_cache.sh vlsd
 compose:
-  tags: [ saas-linux-medium-amd64 ]
+  tags: [saas-linux-medium-amd64]
   stage: test
   variables:
     BUILDER: compose_builder
   before_script:
-    - docker volume create bitcoin_data
-    - docker volume create lightning_data
-    - docker volume create txoo_data
-    - docker volume create vls_data
     - docker buildx create --name=$BUILDER --use
   script:
     - export IMAGE_TAG=${IMAGE_TAG}
@@ -60,3 +56,4 @@ compose:
     - ./scripts/build_from_cache.sh txood
     - ./scripts/build_from_cache.sh vlsd
     - docker compose --profile vls -f docker-compose.yml -f docker-compose.regtest.yml up -d --wait --wait-timeout 120 --build
+    - docker exec vlsd-regtest vls-cli --rpc-user rpcuser --rpc-password VLSsigner2 info | grep -q 101

README.md

@@ -9,26 +9,7 @@ Docker Engine is available on a variety of Linux distros, macOS, and Windows 10
 - [Ubuntu](https://docs.docker.com/engine/install/ubuntu/)
 - [Fedora](https://docs.docker.com/engine/install/fedora/)
 
-### Distro Packages
+**Note**: The compose files present within this repository make use of docker compose v2 the minimum supported version is `v2.26.0`.
-
-Debian/Ubuntu:
-```
-sudo apt install docker.io docker-doc docker-compose containerd runc
-sudo systemctl enable --now docker
-```
-
-Fedora/RHEL:
-```
-sudo dnf install docker docker-compose containerd runc
-sudo systemctl enable --now docker
-```
-
-### Docker v1/v2 Compatibility
-
-- Currently available `docker-compose` packages in different linux distributions (debian, ubuntu, fedora, etc) are not up to date, they are still at version `1` which has been deprecated by `docker` with release of version `2`.
-- `docker-compose` version `2` is available through official docker repositories not the distribution ones.
-- Docker Compose files in this repository work with version 2 and are also __backward compatible__ with version 1.
-- If you are using the distribution installation the `docker compose` command used below has to be changed to `docker-compose` instead.
 
 ## VLS standalone Setup
 
@@ -65,19 +46,28 @@ Required command arguments:
 
 For information on all possible arguments to `vlsd` see [documentation](https://gitlab.com/lightning-signer/validating-lightning-signer/-/blob/main/vls-proxy/src/config.rs?ref_type=heads).
 
+### Build Arguments
+
+For building the container its required to provide following build arguments:
+- `VLS_REPO` url for git repo to use for building binaries.
+- `VLS_GIT_HASH` commit sha/tag/branch to use from git repo.
+- `TXOO_PUBLIC_KEY` public key for trusted TXOO.
+
 ### Running container
 
 There is a `docker-compose.yml` in the `vlsd` folder which can be used to run a standalone `vlsd` service with `network_mode` set to host.
 
 ```bash
 cd vlsd
+export VLS_REPO=$VLS_REPO
 export VLS_GIT_HASH=$VLS_GIT_HASH
+export TXOO_PUBLIC_KEY=$TXOO_PUBLIC_KEY
 docker compose up
 ```
 
-**_Note_**: Make sure to set `BITCOIND_RPC_URL`, `VLS_GIT_HASH` and `CLN_RMEOTE_HSMD_URL` as either environment variables or in the `docker-compose.yml` file before running the above command. You can take `VLS_GIT_HASH` from [.env](./.env) or from the main [repo](https://gitlab.com/lightning-signer/validating-lightning-signer)
+**_Note_**: Make sure to set `BITCOIND_RPC_URL` and `CLN_REMOTE_HSMD_URL` as either environment variables or in the `docker-compose.yml` file before running the above command.
 
-If you wish to run it as a standalone container without using `docker-compose` you can use the following command:
+If you wish to run it as a standalone container without using `docker compose` you can use the following command:
 
 ```bash
 docker run \
@@ -86,6 +76,8 @@ docker run \
   --name vlsd \
   --network host \
   --build_arg VLS_GIT_HASH=$VLS_GIT_HASH
+  --build_arg VLS_REPO=$VLS_REPO
+  --build_arg TXOO_PUBLIC_KEY=$TXOO_PUBLIC_KEY
   -e VLS_NETWORK=testnet \
   -e BITCOIND_RPC_URL=$BITCOIND_RPC_URL \
   --mount 'type=volume,src=vls_data,dst=/home/vls/.lightning-signer' \
@@ -226,8 +218,8 @@ docker compose --profile vls down
 The currently set default versions for services is as follows in the [.env](.env) file:
 - **Bitcoin Core**: v26.0
 - **Core Lightning**: v24.02.2
-- **TXOO**: v0.6.4
+- **TXOO**: v0.8.1
-- **VLS**: v0.11.0
+- **VLS**: v0.12.0
 
 You just can switch to a particular version/commit for a service by updating the git hash and then rebuilding the service:
 ```bash
@@ -251,7 +243,7 @@ Note: For `bitcoind` its also important to update the `BITCOIN_SHA256SUMS_HASH`.
 - [x] Profile configuration to run `vls`
 - [x] Healthcheck for `txoo`
 - [x] Healthcheck for `vls`
-- [ ] Lightning Storage Server Dockerfile and Compose Service
+- [x] Lightning Storage Server Dockerfile and Compose Service
 - [ ] Postgres Service for Lightning Storage Server
 
 ## References

docker-compose.regtest.yml

@@ -31,6 +31,8 @@ services:
 
   txoo:
     container_name: txood-regtest
+    command:
+      - --no-checkpoint
     volumes:
       - txoo_regtest:/root/.txoo/
       - bitcoin_regtest:/root/.bitcoin/
@@ -38,6 +40,15 @@ services:
       - BITCOIN_NETWORK=regtest
       - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:38332
 
+  txoo-server:
+    container_name: txoo-server-regtest
+    volumes:
+      - type: volume
+        source: txoo_regtest
+        target: /usr/share/nginx/html
+        volume:
+          subpath: regtest/public
+
   vls:
     container_name: vlsd-regtest
     volumes:
@@ -46,8 +57,14 @@ services:
       - VLS_NETWORK=regtest
       - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:38332
 
+  lss:
+    container_name: lss-regtest
+    volumes:
+      - lss_regtest:/home/lss/.lss
+
 volumes:
   bitcoin_regtest:
   lightning_regtest:
   txoo_regtest:
   vls_regtest:
+  lss_regtest:

docker-compose.yml

@@ -50,8 +50,11 @@ services:
     depends_on:
       bitcoin-core:
         condition: service_healthy
+      txoo-server:
+        condition: service_started
     environment:
       - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:18332
+      - TXOO_SOURCE_URL=http://txoo-server:80
       - RUST_LOG=info
       - VLS_NETWORK=testnet
       - VLS_BIND=0.0.0.0
@@ -78,6 +81,23 @@ services:
       - BITCOIN_NETWORK=testnet
       - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:18332
 
+  txoo-server:
+    image: nginx:1.27-alpine3.20
+    container_name: txoo-server-test
+    networks:
+      - lightning
+    depends_on:
+      txoo:
+        condition: service_started
+    volumes:
+      - type: volume
+        source: txoo_data
+        target: /usr/share/nginx/html
+        volume:
+          subpath: testnet/public
+    ports:
+      - "8080:80"
+
   lss:
     build:
       context: ./lss
@@ -109,6 +129,7 @@ services:
       args:
         - VLS_REPO
         - VLS_GIT_HASH
+        - TXOO_PUBLIC_KEY
     image: vlsd:${IMAGE_TAG:-latest}
     container_name: vlsd-test
     profiles:
@@ -121,15 +142,18 @@ services:
       - lightning
     volumes:
       - vls_data:/home/vls/.lightning-signer
+    depends_on:
+      txoo-server:
+        condition: service_started
+      lss:
+        condition: service_started
+        required: false
     environment:
+      - LSS_REMOTE_URL
       - VLS_NETWORK=testnet
       - VLS_PERMISSIVE=1
       - RUST_LOG=info
       - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:18332
-    depends_on:
-      lss:
-        condition: service_started
-        required: false
 
 volumes:
   bitcoin_data:

lightningd/Dockerfile

@@ -146,9 +146,9 @@ COPY --from=builder /usr/local/bin/summars /usr/local/src/plugins/summars
 COPY --from=vls_builder /usr/local/bin/remote_hsmd_socket /usr/libexec/c-lightning/remote_hsmd_socket
 COPY --from=builder /build/core-lightning/plugins/clnrest/requirements.txt /usr/local/src/plugins/clnrest_requirements.txt
 
-RUN addgroup -S lightning && adduser -S lightning -G lightning && \
+RUN addgroup -S ${LIGHTNINGD_USER} && adduser -S ${LIGHTNINGD_USER} -G ${LIGHTNINGD_USER} && \
     mkdir -p ${LIGHTNINGD_DATA} && \
-    chown -R lightning:lightning ${LIGHTNINGD_DATA}
+    chown -R ${LIGHTNINGD_USER}:${LIGHTNINGD_USER} ${LIGHTNINGD_DATA}
 
 COPY bitcoin.conf ${BITCOIND_HOME}/bitcoin.conf
 
@@ -161,13 +161,13 @@ RUN chmod +x /healthcheck.sh
 VOLUME ["${LIGHTNINGD_DATA}"]
 
 RUN mkdir -p ${BITCOIND_HOME} && \
-    chown -R lightning:lightning ${BITCOIND_HOME}
+    chown -R ${LIGHTNINGD_USER}:${LIGHTNINGD_USER} ${BITCOIND_HOME}
 
 COPY bitcoin.conf ${BITCOIND_HOME}/bitcoin.conf
 COPY testnet-config /testnet-config
 COPY regtest-config /regtest-config
 
-USER lightning
+USER ${LIGHTNINGD_USER}
 
 RUN pip3 install \
       pyln-client \

lightningd/entrypoint.sh

@@ -4,6 +4,7 @@ set -e
 cp -u /testnet-config ${LIGHTNINGD_DATA}/testnet-config
 cp -u /regtest-config ${LIGHTNINGD_DATA}/regtest-config
 
+# this is kept for backward compatibility purposes
 export GREENLIGHT_VERSION=$(lightningd --version)
 export VLS_CLN_VERSION=$(lightningd --version)
 

scripts/cleanup_regtest.sh

@@ -7,3 +7,4 @@ docker volume rm vls-container_bitcoin_regtest
 docker volume rm vls-container_lightning_regtest
 docker volume rm vls-container_vls_regtest
 docker volume rm vls-container_txoo_regtest
+docker volume ls | grep -q "vls-container_lss_regtest" && docker volume rm vls-container_lss_regtest

txood/Dockerfile

@@ -1,5 +1,5 @@
 # build txoo
-FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as builder
+FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.20 as builder
 
 WORKDIR /build
 
@@ -18,7 +18,7 @@ RUN cd txoo && \
     cargo clean
 
 # txoo runner
-FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as runner
+FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.20 as runner
 
 LABEL maintainer.0="Lakshya Singh (@king-11)" \
   maintainer.1="Dev Random (@devrandom01)"

vlsd/Dockerfile

@@ -24,19 +24,24 @@ RUN cd vls && \
 
 FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as runner
 
-ARG UID=100
+ARG VLSD_UID=100 \
-ARG GID=101
+    VLSD_GID=101 \
+    VLSD_USER=vls
+ARG VLSD_HOME=/home/${VLSD_USER}
+ARG TXOO_PUBLIC_KEY
+ENV TXOO_PUBLIC_KEY=${TXOO_PUBLIC_KEY}
 
 LABEL maintainer.0="Lakshya Singh (@king-11)" \
   maintainer.1="Dev Random (@devrandom01)"
 
-RUN addgroup vls --gid ${GID} --system
+RUN addgroup ${VLSD_USER} --gid ${VLSD_GID} --system
-RUN adduser --uid ${UID} --system vls --ingroup vls
+RUN adduser --uid ${VLSD_UID} --system ${VLSD_USER} --ingroup ${VLSD_USER}
 
 RUN apk update && \
   apk add \
   build-base \
   curl-dev \
+  curl \
   protobuf \
   bind-tools \
   tini
@@ -52,16 +57,18 @@ RUN chmod +x /healthcheck.sh
 
 COPY vlsd2.toml /vlsd2.toml
 
-ENV VLS_DATA=/home/vls/.lightning-signer
+ENV VLSD_DATA=/home/vls/.lightning-signer
-RUN mkdir ${VLS_DATA}
+RUN mkdir -p ${VLSD_DATA} && \
-RUN chown vls:vls ${VLS_DATA}
+    chown ${VLSD_USER}:${VLSD_USER} ${VLSD_DATA}
 
-ENV REMOTE_SIGNER_ALLOWLIST=${VLS_DATA}/ALLOWLIST
+RUN mkdir -p ${VLSD_HOME}/.txoo
+
+ENV REMOTE_SIGNER_ALLOWLIST=${VLSD_DATA}/ALLOWLIST
 RUN touch ${REMOTE_SIGNER_ALLOWLIST}
 
-VOLUME ["${VLS_DATA}"]
+VOLUME ["${VLSD_DATA}"]
 
-USER vls
+USER ${VLSD_USER}
 
 HEALTHCHECK --interval=10s --timeout=10s --start-period=10s \
     CMD ["/healthcheck.sh"]

vlsd/docker-compose.yml

@@ -5,7 +5,9 @@ services:
       context: .
       dockerfile: Dockerfile
       args:
+        - VLS_REPO
         - VLS_GIT_HASH
+        - TXOO_PUBLIC_KEY
     image: vlsd
     container_name: vlsd-standalone
     command:

vlsd/entrypoint.sh

@@ -1,8 +1,12 @@
 #!/bin/sh
 set -e
 
-cp /vlsd2.toml $VLS_DATA/
+cp /vlsd2.toml $VLSD_DATA/
-sed -i "1s/^/network = \"$VLS_NETWORK\"\n/" $VLS_DATA/vlsd2.toml
+sed -i "1s/^/network = \"$VLS_NETWORK\"\n/" $VLSD_DATA/vlsd2.toml
+
+TXOO_PUBLIC_KEY=${TXOO_PUBLIC_KEY:=$(curl -s --retry 5 --retry-all-errors --fail http://txoo-server:80/config | grep public_key | cut -d ' ' -f 2)}
+
+test -n "$TXOO_PUBLIC_KEY" || (echo "TXOO_PUBLIC_KEY build arg not set" && false)
 
 if [ $(echo "$1" | cut -c1) = "-" ]; then
   echo "$0: assuming arguments for vlsd2"
@@ -11,9 +15,10 @@ if [ $(echo "$1" | cut -c1) = "-" ]; then
 fi
 
 if [ $(echo "$1" | cut -c1) = "-" ] || [ "$1" = "vlsd2" ]; then
-  echo "$0: setting config to $VLS_DATA/vlsd2.toml"
+  echo "$0: setting config to $VLSD_DATA/vlsd2.toml"
 
-  set -- "$@" --config=$VLS_DATA/vlsd2.toml
+  echo "$0: using $TXOO_PUBLIC_KEY as trusted oracle pubkey"
+  set -- "$@" --config=$VLSD_DATA/vlsd2.toml -t=$TXOO_PUBLIC_KEY
 fi
 
 echo