Merge branch 'king-11/update-vls' into 'main'

VLS: version v0.12.0 See merge request lightning-signer/vls-container!30
2024-08-29 17:59:26 +00:00 · 2024-08-29 17:59:26 +00:00 · 1df2be3fa5
commit 1df2be3fa5
parent 40f38c9015 00ae8d5728
12 changed files with 119 additions and 73 deletions
--- a/.env
+++ b/.env
@ -4,18 +4,18 @@ BITCOIN_SHA256SUMS_HASH=63487c308a6655f939efd700bfca8e0285fa2f869283aaa7192bdd4b
 # core lightning version v24.05
 CORE_LIGHTNING_REPO=https://github.com/ElementsProject/lightning.git
 CORE_LIGHTNING_GIT_HASH=11586abf79cad33727c03dfa810ae8bcdd3762bf
-# clboss version v0.13.1 (2024-05-31 21:28:55 -0700)
-CLBOSS_REPO=https://github.com/ZmnSCPxj/clboss.git
-CLBOSS_GIT_HASH=df51d5486b05acbeca16cb9e42d18fa082e6079a
-# cln plugins (2024-06-06 17:02:52 +0200)
+# clboss version v0.14.0-rc1
+CLBOSS_REPO=https://github.com/vincenzopalazzo/clboss
+CLBOSS_GIT_HASH=37d493cdf8360c83621d20ac54543b045ae0ef21
+# cln plugins (2024-07-27)
 CLN_PLUGINS_REPO=https://github.com/lightningd/plugins.git
-CLN_PLUGINS_GIT_HASH=de0c7af343df25e82536233ac248ef46efea8670
-# txoo version 0.6.4
+CLN_PLUGINS_GIT_HASH=5e449468bd57db7d0f33178fe0dc867e0da94133
+# txoo version 0.8.1
 TXOO_REPO=https://gitlab.com/lightning-signer/txoo.git
-TXOO_GIT_HASH=6f0718e3f2b9406df5e3cd73306f473199141da0
-# vls version v0.11.0 (2024-06-06 15:09:25 +0000)
+TXOO_GIT_HASH=d2f698c1434a91d5d53be4fd96ff01d3c5dfa22a
+# vls version v0.12.0
 VLS_REPO=https://gitlab.com/lightning-signer/validating-lightning-signer.git
-VLS_GIT_HASH=d2590ba34a388a016bb91307864e993fb3cc3d84
+VLS_GIT_HASH=c17067773b0bc57b27a6a0c7127c48867563e280
 # lss version v0.1.0
 LSS_REPO=https://gitlab.com/lightning-signer/validating-lightning-signer.git
 LSS_GIT_HASH=d2590ba34a388a016bb91307864e993fb3cc3d84
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -3,9 +3,9 @@ stages:
  - test

 default:
-  image: docker:24.0.7
+  image: docker:26.0.1
  services:
-    - docker:24.0.7-dind
+    - docker:26.0.1-dind
  before_script:
    - docker version
    - docker compose version
@ -18,17 +18,17 @@ variables:
  IMAGE_TAG: $CI_COMMIT_SHA

 build_bitcoin:
-  tags: [ saas-linux-large-amd64 ]
+  tags: [saas-linux-large-amd64]
  stage: build
  script:
    - ./scripts/build_image_cache.sh bitcoind
 build_core_lightning:
-  tags: [ saas-linux-large-amd64 ]
+  tags: [saas-linux-large-amd64]
  stage: build
  script:
    - ./scripts/build_image_cache.sh lightningd
 build_txoo:
-  tags: [ saas-linux-medium-amd64 ]
+  tags: [saas-linux-medium-amd64]
  stage: build
  script:
    - ./scripts/build_image_cache.sh txood
@ -38,20 +38,16 @@ build_lss:
  script:
    - ./scripts/build_image_cache.sh lss
 build_vls:
-  tags: [ saas-linux-large-amd64 ]
+  tags: [saas-linux-large-amd64]
  stage: build
  script:
    - ./scripts/build_image_cache.sh vlsd
 compose:
-  tags: [ saas-linux-medium-amd64 ]
+  tags: [saas-linux-medium-amd64]
  stage: test
  variables:
    BUILDER: compose_builder
  before_script:
-    - docker volume create bitcoin_data
-    - docker volume create lightning_data
-    - docker volume create txoo_data
-    - docker volume create vls_data
    - docker buildx create --name=$BUILDER --use
  script:
    - export IMAGE_TAG=${IMAGE_TAG}
@ -60,3 +56,4 @@ compose:
    - ./scripts/build_from_cache.sh txood
    - ./scripts/build_from_cache.sh vlsd
    - docker compose --profile vls -f docker-compose.yml -f docker-compose.regtest.yml up -d --wait --wait-timeout 120 --build
+    - docker exec vlsd-regtest vls-cli --rpc-user rpcuser --rpc-password VLSsigner2 info | grep -q 101
--- a/README.md
+++ b/README.md
@ -9,26 +9,7 @@ Docker Engine is available on a variety of Linux distros, macOS, and Windows 10
 - [Ubuntu](https://docs.docker.com/engine/install/ubuntu/)
 - [Fedora](https://docs.docker.com/engine/install/fedora/)

-### Distro Packages
-
-Debian/Ubuntu:
-```
-sudo apt install docker.io docker-doc docker-compose containerd runc
-sudo systemctl enable --now docker
-```
-
-Fedora/RHEL:
-```
-sudo dnf install docker docker-compose containerd runc
-sudo systemctl enable --now docker
-```
-
-### Docker v1/v2 Compatibility
-
- Currently available `docker-compose` packages in different linux distributions (debian, ubuntu, fedora, etc) are not up to date, they are still at version `1` which has been deprecated by `docker` with release of version `2`.
- `docker-compose` version `2` is available through official docker repositories not the distribution ones.
- Docker Compose files in this repository work with version 2 and are also __backward compatible__ with version 1.
- If you are using the distribution installation the `docker compose` command used below has to be changed to `docker-compose` instead.
+**Note**: The compose files present within this repository make use of docker compose v2 the minimum supported version is `v2.26.0`.

 ## VLS standalone Setup

@ -65,19 +46,28 @@ Required command arguments:

 For information on all possible arguments to `vlsd` see [documentation](https://gitlab.com/lightning-signer/validating-lightning-signer/-/blob/main/vls-proxy/src/config.rs?ref_type=heads).

+### Build Arguments
+
+For building the container its required to provide following build arguments:
+- `VLS_REPO` url for git repo to use for building binaries.
+- `VLS_GIT_HASH` commit sha/tag/branch to use from git repo.
+- `TXOO_PUBLIC_KEY` public key for trusted TXOO.
+
 ### Running container

 There is a `docker-compose.yml` in the `vlsd` folder which can be used to run a standalone `vlsd` service with `network_mode` set to host.

 ```bash
 cd vlsd
+export VLS_REPO=$VLS_REPO
 export VLS_GIT_HASH=$VLS_GIT_HASH
+export TXOO_PUBLIC_KEY=$TXOO_PUBLIC_KEY
 docker compose up
 ```

-**_Note_**: Make sure to set `BITCOIND_RPC_URL`, `VLS_GIT_HASH` and `CLN_RMEOTE_HSMD_URL` as either environment variables or in the `docker-compose.yml` file before running the above command. You can take `VLS_GIT_HASH` from [.env](./.env) or from the main [repo](https://gitlab.com/lightning-signer/validating-lightning-signer)
+**_Note_**: Make sure to set `BITCOIND_RPC_URL` and `CLN_REMOTE_HSMD_URL` as either environment variables or in the `docker-compose.yml` file before running the above command.

-If you wish to run it as a standalone container without using `docker-compose` you can use the following command:
+If you wish to run it as a standalone container without using `docker compose` you can use the following command:

 ```bash
 docker run \
@ -86,6 +76,8 @@ docker run \
  --name vlsd \
  --network host \
  --build_arg VLS_GIT_HASH=$VLS_GIT_HASH
+  --build_arg VLS_REPO=$VLS_REPO
+  --build_arg TXOO_PUBLIC_KEY=$TXOO_PUBLIC_KEY
  -e VLS_NETWORK=testnet \
  -e BITCOIND_RPC_URL=$BITCOIND_RPC_URL \
  --mount 'type=volume,src=vls_data,dst=/home/vls/.lightning-signer' \
@ -226,8 +218,8 @@ docker compose --profile vls down
 The currently set default versions for services is as follows in the [.env](.env) file:
 - **Bitcoin Core**: v26.0
 - **Core Lightning**: v24.02.2
- **TXOO**: v0.6.4
- **VLS**: v0.11.0
+- **TXOO**: v0.8.1
+- **VLS**: v0.12.0

 You just can switch to a particular version/commit for a service by updating the git hash and then rebuilding the service:
 ```bash
@ -251,7 +243,7 @@ Note: For `bitcoind` its also important to update the `BITCOIN_SHA256SUMS_HASH`.
 - [x] Profile configuration to run `vls`
 - [x] Healthcheck for `txoo`
 - [x] Healthcheck for `vls`
- [ ] Lightning Storage Server Dockerfile and Compose Service
+- [x] Lightning Storage Server Dockerfile and Compose Service
 - [ ] Postgres Service for Lightning Storage Server

 ## References
--- a/docker-compose.regtest.yml
+++ b/docker-compose.regtest.yml
@ -31,6 +31,8 @@ services:

  txoo:
    container_name: txood-regtest
+    command:
+      - --no-checkpoint
    volumes:
      - txoo_regtest:/root/.txoo/
      - bitcoin_regtest:/root/.bitcoin/
@ -38,6 +40,15 @@ services:
      - BITCOIN_NETWORK=regtest
      - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:38332

+  txoo-server:
+    container_name: txoo-server-regtest
+    volumes:
+      - type: volume
+        source: txoo_regtest
+        target: /usr/share/nginx/html
+        volume:
+          subpath: regtest/public
+
  vls:
    container_name: vlsd-regtest
    volumes:
@ -46,8 +57,14 @@ services:
      - VLS_NETWORK=regtest
      - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:38332

+  lss:
+    container_name: lss-regtest
+    volumes:
+      - lss_regtest:/home/lss/.lss
+
 volumes:
  bitcoin_regtest:
  lightning_regtest:
  txoo_regtest:
  vls_regtest:
+  lss_regtest:
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -50,8 +50,11 @@ services:
    depends_on:
      bitcoin-core:
        condition: service_healthy
+      txoo-server:
+        condition: service_started
    environment:
      - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:18332
+      - TXOO_SOURCE_URL=http://txoo-server:80
      - RUST_LOG=info
      - VLS_NETWORK=testnet
      - VLS_BIND=0.0.0.0
@ -78,6 +81,23 @@ services:
      - BITCOIN_NETWORK=testnet
      - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:18332

+  txoo-server:
+    image: nginx:1.27-alpine3.20
+    container_name: txoo-server-test
+    networks:
+      - lightning
+    depends_on:
+      txoo:
+        condition: service_started
+    volumes:
+      - type: volume
+        source: txoo_data
+        target: /usr/share/nginx/html
+        volume:
+          subpath: testnet/public
+    ports:
+      - "8080:80"
+
  lss:
    build:
      context: ./lss
@ -109,6 +129,7 @@ services:
      args:
        - VLS_REPO
        - VLS_GIT_HASH
+        - TXOO_PUBLIC_KEY
    image: vlsd:${IMAGE_TAG:-latest}
    container_name: vlsd-test
    profiles:
@ -121,15 +142,18 @@ services:
      - lightning
    volumes:
      - vls_data:/home/vls/.lightning-signer
+    depends_on:
+      txoo-server:
+        condition: service_started
+      lss:
+        condition: service_started
+        required: false
    environment:
+      - LSS_REMOTE_URL
      - VLS_NETWORK=testnet
      - VLS_PERMISSIVE=1
      - RUST_LOG=info
      - BITCOIND_RPC_URL=http://rpcuser:VLSsigner1@bitcoind:18332
-    depends_on:
-      lss:
-        condition: service_started
-        required: false

 volumes:
  bitcoin_data:
--- a/lightningd/Dockerfile
+++ b/lightningd/Dockerfile
@ -146,9 +146,9 @@ COPY --from=builder /usr/local/bin/summars /usr/local/src/plugins/summars
 COPY --from=vls_builder /usr/local/bin/remote_hsmd_socket /usr/libexec/c-lightning/remote_hsmd_socket
 COPY --from=builder /build/core-lightning/plugins/clnrest/requirements.txt /usr/local/src/plugins/clnrest_requirements.txt

-RUN addgroup -S lightning && adduser -S lightning -G lightning && \
+RUN addgroup -S ${LIGHTNINGD_USER} && adduser -S ${LIGHTNINGD_USER} -G ${LIGHTNINGD_USER} && \
    mkdir -p ${LIGHTNINGD_DATA} && \
-    chown -R lightning:lightning ${LIGHTNINGD_DATA}
+    chown -R ${LIGHTNINGD_USER}:${LIGHTNINGD_USER} ${LIGHTNINGD_DATA}

 COPY bitcoin.conf ${BITCOIND_HOME}/bitcoin.conf

@ -161,13 +161,13 @@ RUN chmod +x /healthcheck.sh
 VOLUME ["${LIGHTNINGD_DATA}"]

 RUN mkdir -p ${BITCOIND_HOME} && \
-    chown -R lightning:lightning ${BITCOIND_HOME}
+    chown -R ${LIGHTNINGD_USER}:${LIGHTNINGD_USER} ${BITCOIND_HOME}

 COPY bitcoin.conf ${BITCOIND_HOME}/bitcoin.conf
 COPY testnet-config /testnet-config
 COPY regtest-config /regtest-config

-USER lightning
+USER ${LIGHTNINGD_USER}

 RUN pip3 install \
      pyln-client \
--- a/lightningd/entrypoint.sh
+++ b/lightningd/entrypoint.sh
@ -4,6 +4,7 @@ set -e
 cp -u /testnet-config ${LIGHTNINGD_DATA}/testnet-config
 cp -u /regtest-config ${LIGHTNINGD_DATA}/regtest-config

+# this is kept for backward compatibility purposes
 export GREENLIGHT_VERSION=$(lightningd --version)
 export VLS_CLN_VERSION=$(lightningd --version)

--- a/scripts/cleanup_regtest.sh
+++ b/scripts/cleanup_regtest.sh
@ -7,3 +7,4 @@ docker volume rm vls-container_bitcoin_regtest
 docker volume rm vls-container_lightning_regtest
 docker volume rm vls-container_vls_regtest
 docker volume rm vls-container_txoo_regtest
+docker volume ls | grep -q "vls-container_lss_regtest" && docker volume rm vls-container_lss_regtest
--- a/txood/Dockerfile
+++ b/txood/Dockerfile
@ -1,5 +1,5 @@
 # build txoo
-FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as builder
+FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.20 as builder

 WORKDIR /build

@ -18,7 +18,7 @@ RUN cd txoo && \
    cargo clean

 # txoo runner
-FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as runner
+FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.20 as runner

 LABEL maintainer.0="Lakshya Singh (@king-11)" \
  maintainer.1="Dev Random (@devrandom01)"
--- a/vlsd/Dockerfile
+++ b/vlsd/Dockerfile
@ -24,19 +24,24 @@ RUN cd vls && \

 FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as runner

-ARG UID=100
-ARG GID=101
+ARG VLSD_UID=100 \
+    VLSD_GID=101 \
+    VLSD_USER=vls
+ARG VLSD_HOME=/home/${VLSD_USER}
+ARG TXOO_PUBLIC_KEY
+ENV TXOO_PUBLIC_KEY=${TXOO_PUBLIC_KEY}

 LABEL maintainer.0="Lakshya Singh (@king-11)" \
  maintainer.1="Dev Random (@devrandom01)"

-RUN addgroup vls --gid ${GID} --system
-RUN adduser --uid ${UID} --system vls --ingroup vls
+RUN addgroup ${VLSD_USER} --gid ${VLSD_GID} --system
+RUN adduser --uid ${VLSD_UID} --system ${VLSD_USER} --ingroup ${VLSD_USER}

 RUN apk update && \
  apk add \
  build-base \
  curl-dev \
+  curl \
  protobuf \
  bind-tools \
  tini
@ -52,16 +57,18 @@ RUN chmod +x /healthcheck.sh

 COPY vlsd2.toml /vlsd2.toml

-ENV VLS_DATA=/home/vls/.lightning-signer
-RUN mkdir ${VLS_DATA}
-RUN chown vls:vls ${VLS_DATA}
+ENV VLSD_DATA=/home/vls/.lightning-signer
+RUN mkdir -p ${VLSD_DATA} && \
+    chown ${VLSD_USER}:${VLSD_USER} ${VLSD_DATA}

-ENV REMOTE_SIGNER_ALLOWLIST=${VLS_DATA}/ALLOWLIST
+RUN mkdir -p ${VLSD_HOME}/.txoo
+
+ENV REMOTE_SIGNER_ALLOWLIST=${VLSD_DATA}/ALLOWLIST
 RUN touch ${REMOTE_SIGNER_ALLOWLIST}

-VOLUME ["${VLS_DATA}"]
+VOLUME ["${VLSD_DATA}"]

-USER vls
+USER ${VLSD_USER}

 HEALTHCHECK --interval=10s --timeout=10s --start-period=10s \
    CMD ["/healthcheck.sh"]
--- a/vlsd/docker-compose.yml
+++ b/vlsd/docker-compose.yml
@ -5,7 +5,9 @@ services:
      context: .
      dockerfile: Dockerfile
      args:
+        - VLS_REPO
        - VLS_GIT_HASH
+        - TXOO_PUBLIC_KEY
    image: vlsd
    container_name: vlsd-standalone
    command:
--- a/vlsd/entrypoint.sh
+++ b/vlsd/entrypoint.sh
@ -1,8 +1,12 @@
 #!/bin/sh
 set -e

-cp /vlsd2.toml $VLS_DATA/
-sed -i "1s/^/network = \"$VLS_NETWORK\"\n/" $VLS_DATA/vlsd2.toml
+cp /vlsd2.toml $VLSD_DATA/
+sed -i "1s/^/network = \"$VLS_NETWORK\"\n/" $VLSD_DATA/vlsd2.toml
+
+TXOO_PUBLIC_KEY=${TXOO_PUBLIC_KEY:=$(curl -s --retry 5 --retry-all-errors --fail http://txoo-server:80/config | grep public_key | cut -d ' ' -f 2)}
+
+test -n "$TXOO_PUBLIC_KEY" || (echo "TXOO_PUBLIC_KEY build arg not set" && false)

 if [ $(echo "$1" | cut -c1) = "-" ]; then
  echo "$0: assuming arguments for vlsd2"
@ -11,9 +15,10 @@ if [ $(echo "$1" | cut -c1) = "-" ]; then
 fi

 if [ $(echo "$1" | cut -c1) = "-" ] || [ "$1" = "vlsd2" ]; then
-  echo "$0: setting config to $VLS_DATA/vlsd2.toml"
+  echo "$0: setting config to $VLSD_DATA/vlsd2.toml"

-  set -- "$@" --config=$VLS_DATA/vlsd2.toml
+  echo "$0: using $TXOO_PUBLIC_KEY as trusted oracle pubkey"
+  set -- "$@" --config=$VLSD_DATA/vlsd2.toml -t=$TXOO_PUBLIC_KEY
 fi

 echo