cln: update to v24.05 clboss to master

add jq as new dependency
Signed-off-by: Lakshya Singh <lakshay.singh1108@gmail.com>

.env

@@ -1,15 +1,15 @@
 # bitcoin version 26.0
 BITCOIN_VERSION=26.0
 BITCOIN_SHA256SUMS_HASH=63487c308a6655f939efd700bfca8e0285fa2f869283aaa7192bdd4b8799a747
-# core lightning version v24.02
+# core lightning version v24.05
 CORE_LIGHTNING_REPO=https://github.com/ElementsProject/lightning.git
-CORE_LIGHTNING_GIT_HASH=8418989f9bf3563a67ac91602500672b51628579
-# clboss
+CORE_LIGHTNING_GIT_HASH=11586abf79cad33727c03dfa810ae8bcdd3762bf
+# clboss version v0.13.1 (2024-05-31 21:28:55 -0700)
 CLBOSS_REPO=https://github.com/ZmnSCPxj/clboss.git
-CLBOSS_GIT_HASH=0673c50e7374ea8f5cb7e302f72b7978c6bd1794
-# cln plugins
+CLBOSS_GIT_HASH=df51d5486b05acbeca16cb9e42d18fa082e6079a
+# cln plugins (2024-06-06 17:02:52 +0200)
 CLN_PLUGINS_REPO=https://github.com/lightningd/plugins.git
-CLN_PLUGINS_GIT_HASH=a525e6c42033a6270c81065a559301ceb2a761fa
+CLN_PLUGINS_GIT_HASH=de0c7af343df25e82536233ac248ef46efea8670
 # txoo version 0.6.4
 TXOO_REPO=https://gitlab.com/lightning-signer/txoo.git
 TXOO_GIT_HASH=6f0718e3f2b9406df5e3cd73306f473199141da0

README.md

@@ -138,21 +138,16 @@ __Note__: Even while using `testnet` running using the override is recommended a
 To run a single node without `vls` service we can use the same `docker-compose.yml` file in the main directory by just removing the `profile` flag `vls` from all commands.
 
 ```bash
-docker compose up --build
+docker compose -f docker-compose.yml -f $DOCKER_COMPOSE_OVERRIDE up --build
 ```
 
 Above command will run `bitcoind`, `lightningd` and `txood` services on a single node.
 
-### Additional Regtest Commands
+## Interacting with Containers
 
-We have to run these commands after bitcoind is up and running.
+We can use the `docker container exec <CONTAINER_NAME> <COMMAND>` command to interact with containers and interact with setup using command line tools like `bitcoin-cli`, `lightning-cli`, `vls-cli`, etc.
 
-Create wallet and make it load automatically:
-```bash
-docker container exec bitcoind-regtest bitcoin-cli createwallet default
-docker container exec bitcoind-regtest bitcoin-cli unloadwallet default
-docker container exec bitcoind-regtest bitcoin-cli loadwallet default true
-```
+### Regtest Commands
 
 Generate Address for node:
 ```bash
@@ -183,11 +178,54 @@ docker container exec lightningd-test lightning-cli --testnet clboss-status | le
 # Show node summary status
 docker container exec lightningd-test lightning-cli --testnet summary
 ```
+
+## Debugging Containers
+
+Checking running status and health of containers
+```bash
+docker ps
+```
+
+Getting logs from container
+```bash
+# CONTAINER_NAME=lightningd-test
+docker container logs $CONTAINER_NAME > $CONTAINER_NAME.log
+```
+
+Restarting containers
+```bash
+cd vls-container
+docker compose --profile vls stop
+# to start testnet
+docker compose --profile vls up
+```
+
+Generating Backtrace from CLN core dump
+```bash
+# attach to the lightningd container
+docker container exec -u root -it lightningd-test sh
+
+# install gdb
+apk add gdb
+gdb /usr/libexec/c-lightning/plugins/pay /home/lightning/.lightning/testnet/core
+
+# get backtrace
+bt
+```
+
+Delete containers
+```bash
+cd vls-container
+docker compose --profile vls down
+```
+
+**NOTE**: If you want to start fresh make sure to delete the created docker volumes as well.
+
 ## Choosing Versions
 
 The currently set default versions for services is as follows in the [.env](.env) file:
 - **Bitcoin Core**: v26.0
-- **Core Lightning**: v24.02
+- **Core Lightning**: v24.02.2
 - **TXOO**: v0.6.4
 - **VLS**: v0.11.0
 

bitcoind/Dockerfile

@@ -25,30 +25,27 @@ RUN mkdir $GNUPGHOME && set -ex \
     152812300785C96444D3334D17565732E08E5E41 \
     0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 \
     590B7292695AFFA5B672CBB2E13FC145CD3F4304 \
-    28F5900B1BB5D1A4B6B6D1A9ED357015286A333D \
-    637DB1E23370F84AFF88CCE03152347D07DA627C \
     CFB16E21C950F67FA95E558F2EEB9F5CC09526C1 \
     F4FC70F07310028424EFC20A8E4256593F177720 \
     D1DBF2C4B96F2DEBF4C16654410108112E7EA81F \
     287AE4CA1187C68C08B49CB2D11BD4F33F1DB499 \
-    F9A8737BF4FF5C89C903DF31DD78544CF91B1514 \
     9DEAE0DC7063249FB05474681E4AED62986CD25D \
-    E463A93F5F3117EEDE6C7316BD02942421F4889F \
-    9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C \
-    4DAF18FE948E7A965B30F9457E296D555E7F63A7 \
+    3EB0DEE6004A13BE5A0CC758BF2978B068054311 \
+    ED9BDF7AD6A55E232E84524257FF9BDBCC301009 \
     28E72909F1717FE9607754F8A7BEB2621678D37D \
-    74E2DEF5D77260B98BC19438099BAD163C70FBFA \
   ; do \
     gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" || \
     gpg --batch --keyserver keys.openpgp.org --recv-keys "$key" || \
+    gpg --batch --keyserver pgp.mit.edu --recv-keys "$key" || \
     gpg --batch --keyserver keyserver.pgp.com --recv-keys "$key" || \
     gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" || \
     gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" ; \
-  done && \
-  wget -O- https://raw.githubusercontent.com/Kvaciral/kvaciral/main/kvaciral.asc | gpg --import
-
+  done
+RUN gpg --keyserver hkps://keys.openpgp.org --refresh-keys
 RUN gpg -kv
 
+COPY verify.sh verify.sh
+
 ARG BITCOIN_VERSION
 ARG BITCOIN_PREFIX=/opt/bitcoin-${BITCOIN_VERSION}
 ARG BITCOIN_SHA256SUMS_HASH
@@ -56,16 +53,16 @@ RUN echo "Building bitcoin core version $BITCOIN_VERSION"
 RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS && \
     wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS.asc && \
     wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}.tar.gz && \
-    gpg --batch --verify SHA256SUMS.asc SHA256SUMS && \
+    ./verify.sh && \
     echo "${BITCOIN_SHA256SUMS_HASH} SHA256SUMS" | sha256sum -c - && \
     grep " bitcoin-${BITCOIN_VERSION}.tar.gz\$" SHA256SUMS | sha256sum -c - && \
     tar -xzf *.tar.gz
 
 WORKDIR /bitcoin-${BITCOIN_VERSION}
 
-RUN sed -i '/AC_PREREQ/a\AR_FLAGS=cr' src/univalue/configure.ac && \
+RUN sed -i '/AC_PREREQ/a\AR_FLAGS=cr' configure.ac && \
     sed -i '/AX_PROG_CC_FOR_BUILD/a\AR_FLAGS=cr' src/secp256k1/configure.ac && \
-    sed -i s:sys/fcntl.h:fcntl.h: src/compat.h && \
+    sed -i s:sys/fcntl.h:fcntl.h: src/compat/compat.h && \
     ./autogen.sh && \
     ./configure LDFLAGS=-L`ls -d /opt/db*`/lib/ CPPFLAGS=-I`ls -d /opt/db*`/include/ \
     --prefix=${BITCOIN_PREFIX} \
@@ -106,7 +103,8 @@ RUN apk --no-cache add \
   boost-thread \
   sqlite-dev \
   libevent \
-  libzmq
+  libzmq \
+  tini
 
 ARG BITCOIN_VERSION
 ARG BITCOIN_PREFIX=/opt/bitcoin-${BITCOIN_VERSION}
@@ -132,7 +130,7 @@ VOLUME ["${BITCOIN_DATA}"]
 
 USER bitcoin
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"]
 
 RUN bitcoind -version | grep "Bitcoin Core version v${BITCOIN_VERSION}"
 

bitcoind/verify.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# verify SHA256SUMS
+gpg --batch --verify SHA256SUMS.asc SHA256SUMS
+ret_val=$?
+
+# allow 2 as well in case of untrusted keys
+if [ $ret_val -eq 0 ] || [ $ret_val -eq 2 ]; then
+  exit 0
+else
+  exit $ret_val
+fi

docker-compose.yml

@@ -43,6 +43,8 @@ services:
     expose:
       - 19735
       - 7701
+    ports:
+      - 3010:3010
     networks:
       - lightning
     depends_on:

lightningd/Dockerfile

@@ -15,17 +15,22 @@ RUN apk update && \
       gettext \
       git \
       gmp-dev \
+      jq \
       libsodium \
       libtool \
       net-tools \
       postgresql-dev \
       py3-mako \
+      py3-pip \
       python3 \
       python3-dev \
       sqlite-dev \
       sqlite-static \
       zlib-dev \
-      zlib-static
+      zlib-static \
+      linux-headers
+
+RUN pip3 install grpcio-tools
 
 ARG CORE_LIGHTNING_GIT_HASH \
     CORE_LIGHTNING_REPO
@@ -33,7 +38,6 @@ RUN git clone --recursive $CORE_LIGHTNING_REPO core-lightning
 RUN cd core-lightning && \
     git checkout $CORE_LIGHTNING_GIT_HASH && \
     ./configure --enable-static --prefix=/usr && \
-    make -j $(nproc) && \
     make -j $(nproc) install && \
     make clean
 
@@ -41,7 +45,13 @@ ARG CLN_PLUGINS_GIT_HASH \
     CLN_PLUGINS_REPO
 RUN git clone $CLN_PLUGINS_REPO
 RUN cd plugins && \
-    git checkout $CLN_PLUGINS_GIT_HASH
+    git checkout $CLN_PLUGINS_GIT_HASH && \
+    git submodule update --init --recursive
+
+# Build summars plugin
+RUN cd plugins/summars && \
+    cargo install --locked --path . --bin summars --profile release --root /usr/local/ && \
+    cargo clean
 
 FROM --platform=${TARGETPLATFORM:-${BUILDPLATFORM:-linux/amd64}} alpine:3.18 as clboss_builder
 
@@ -111,7 +121,10 @@ RUN apk update && \
       bind-tools \
       libev-dev \
       curl-dev \
-      sqlite-dev
+      sqlite-dev \
+      tini \
+      python3-dev \
+      libffi-dev
 
 ARG LIGHTNINGD_UID=101 \
     LIGHTNINGD_USER=lightning
@@ -128,8 +141,10 @@ COPY --from=builder /usr/libexec/c-lightning /usr/libexec/c-lightning
 COPY --from=builder /usr/share/man/man8 /usr/share/man/man8
 COPY --from=builder /usr/share/doc/c-lightning /usr/share/doc/c-lightning
 COPY --from=clboss_builder /usr/bin/clboss /usr/bin/clboss
-COPY --from=builder /build/plugins/summary /usr/local/src/plugins/summary
+COPY --from=builder /build/plugins/monitor/monitor.py /usr/local/src/plugins/monitor.py
+COPY --from=builder /usr/local/bin/summars /usr/local/src/plugins/summars
 COPY --from=vls_builder /usr/local/bin/remote_hsmd_socket /usr/libexec/c-lightning/remote_hsmd_socket
+COPY --from=builder /build/core-lightning/plugins/clnrest/requirements.txt /usr/local/src/plugins/clnrest_requirements.txt
 
 RUN addgroup -S lightning && adduser -S lightning -G lightning && \
     mkdir -p ${LIGHTNINGD_DATA} && \
@@ -159,8 +174,10 @@ RUN pip3 install \
       requests \
       packaging
 
+RUN pip3 install -r /usr/local/src/plugins/clnrest_requirements.txt
+
 HEALTHCHECK --interval=5s --timeout=10s --start-period=5s \
     CMD ["/healthcheck.sh"]
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"]
 CMD ["lightningd"]

lightningd/entrypoint.sh

@@ -5,6 +5,7 @@ cp -u /testnet-config ${LIGHTNINGD_DATA}/testnet-config
 cp -u /regtest-config ${LIGHTNINGD_DATA}/regtest-config
 
 export GREENLIGHT_VERSION=$(lightningd --version)
+export VLS_CLN_VERSION=$(lightningd --version)
 
 if [ $(echo "$1" | cut -c1) = "-" ]; then
   echo "$0: assuming arguments for lightningd"

lightningd/healthcheck.sh

@@ -2,4 +2,4 @@
 
 set -ex
 
-lightning-cli --network $VLS_NETWORK summary
+lightning-cli --network $VLS_NETWORK getinfo

lightningd/regtest-config

@@ -3,10 +3,11 @@ bitcoin-rpcuser=rpcuser
 bitcoin-rpcpassword=VLSsigner1
 bitcoin-rpcport=38332
 log-level=info
-max-locktime-blocks=288
 important-plugin=/usr/bin/clboss
 clboss-auto-close=true
 bind-addr=0.0.0.0:19846
-plugin=/usr/local/src/plugins/summary/summary.py
+plugin=/usr/local/src/plugins/summars
+plugin=/usr/local/src/plugins/monitor.py
 subdaemon=hsmd:remote_hsmd_socket
-experimental-anchors
+experimental-anchors
+clnrest-port=3010

lightningd/testnet-config

@@ -3,10 +3,11 @@ bitcoin-rpcuser=rpcuser
 bitcoin-rpcpassword=VLSsigner1
 bitcoin-rpcport=18332
 log-level=info
-max-locktime-blocks=288
 important-plugin=/usr/bin/clboss
 clboss-auto-close=true
 bind-addr=0.0.0.0:19735
-plugin=/usr/local/src/plugins/summary/summary.py
+plugin=/usr/local/src/plugins/summars
+plugin=/usr/local/src/plugins/monitor.py
 subdaemon=hsmd:remote_hsmd_socket
-experimental-anchors
+experimental-anchors
+clnrest-port=3010

txood/Dockerfile

@@ -32,7 +32,8 @@ RUN apk update && \
       libev-dev \
       curl-dev \
       curl \
-      jq
+      jq \
+      tini
 
 COPY --from=builder /usr/local/bin/txood /usr/bin/txood
 
@@ -47,5 +48,5 @@ VOLUME ["${TXOO_DATA}"]
 HEALTHCHECK --interval=5s --timeout=10s --start-period=5s \
     CMD ["/healthcheck.sh"]
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"]
 CMD ["txood"]

vlsd/Dockerfile

@@ -38,7 +38,8 @@ RUN apk update && \
   build-base \
   curl-dev \
   protobuf \
-  bind-tools
+  bind-tools \
+  tini
 
 COPY --from=builder /usr/local/bin/vlsd2 /usr/local/bin/vlsd2
 COPY --from=builder /usr/local/bin/vls-cli /usr/local/bin/vls-cli
@@ -65,7 +66,7 @@ USER vls
 HEALTHCHECK --interval=10s --timeout=10s --start-period=10s \
     CMD ["/healthcheck.sh"]
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/entrypoint.sh"]
 
 RUN vlsd2 --git-desc