From 2d8f4c94eb22e6f109700f9af0ee9b8dbdbf0c96 Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Sun, 12 Nov 2023 11:27:12 +0530 Subject: [PATCH 1/5] vls: standalone docker image for separate build Signed-off-by: Lakshya Singh --- README.md | 7 +++++++ docker-compose.yml | 3 +-- vlsd/Dockerfile | 4 ++-- {assets => vlsd}/vlsd2.toml | 0 4 files changed, 10 insertions(+), 4 deletions(-) rename {assets => vlsd}/vlsd2.toml (100%) diff --git a/README.md b/README.md index 135f4cd..5e778de 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,13 @@ sudo systemctl enable --now docker - Docker Images in this repository work with version 2 and are also __backward compatible__ with version 1. - If you are using the distribution installation the `docker compose` command used below has to be changed to `docker-compose` instead. +## VLSD Image + +``` +cd vlsd +docker build -t vlsd . +``` + ## Volume Creation ``` diff --git a/docker-compose.yml b/docker-compose.yml index 0543418..c55e8fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,8 +66,7 @@ services: vls: build: - dockerfile: ./vlsd/Dockerfile - context: . + context: ./vlsd image: vlsd container_name: vlsd-test command: diff --git a/vlsd/Dockerfile b/vlsd/Dockerfile index f361cea..7705576 100644 --- a/vlsd/Dockerfile +++ b/vlsd/Dockerfile @@ -40,10 +40,10 @@ RUN apk update && \ COPY --from=builder /build/validating-lightning-signer/target/release/vlsd2 /usr/local/bin/vlsd2 -COPY vlsd/entrypoint.sh /entrypoint.sh +COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh -COPY assets/vlsd2.toml /vlsd2.toml +COPY vlsd2.toml /vlsd2.toml ENV VLS_DATA=/home/vls/.lightning-signer RUN mkdir "${VLS_DATA}" diff --git a/assets/vlsd2.toml b/vlsd/vlsd2.toml similarity index 100% rename from assets/vlsd2.toml rename to vlsd/vlsd2.toml From bac309d7586525992e9dbd0cb38dc13677d7531e Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Sun, 12 Nov 2023 21:56:57 +0530 Subject: [PATCH 2/5] vls: standalone docker compose Signed-off-by: Lakshya Singh --- vlsd/docker-compose.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 vlsd/docker-compose.yml diff --git a/vlsd/docker-compose.yml b/vlsd/docker-compose.yml new file mode 100644 index 0000000..ac067c3 --- /dev/null +++ b/vlsd/docker-compose.yml @@ -0,0 +1,21 @@ +version: "3.8" +services: + vls: + build: + context: . + dockerfile: Dockerfile + image: vlsd + container_name: vlsd-standalone + command: + - --connect=$CLN_REMOTE_HSMD_URL + network_mode: host + volumes: + - vls_data:/home/vls/.lightning-signer + environment: + - BITCOIND_RPC_URL=$BITCOIND_RPC_URL + - VLS_NETWORK=testnet + +volumes: + vls_data: + name: vls_data + external: true From 3de4e116c960dd6994fa123f5fb1625ae01caced Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Sun, 12 Nov 2023 21:59:32 +0530 Subject: [PATCH 3/5] readme: vls and compose setup - for running standalone vls - fix heading size and grammar - improve description - add command highlighting Signed-off-by: Lakshya Singh --- README.md | 114 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 94 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5e778de..350cc91 100644 --- a/README.md +++ b/README.md @@ -23,71 +23,135 @@ sudo dnf install docker docker-compose containerd runc sudo systemctl enable --now docker ``` -### Docker Compose Compatibility +### Docker v1/v2 Compatibility -- Currently available `docker-compose` package in different linux distributions (debian, ubuntu, fedora, etc) are not up to date, they are still at version `1` which has been deprecated by `docker` with release of version `2`. +- Currently available `docker-compose` packages in different linux distributions (debian, ubuntu, fedora, etc) are not up to date, they are still at version `1` which has been deprecated by `docker` with release of version `2`. - `docker-compose` version `2` is available through official docker repositories not the distribution ones. -- Docker Images in this repository work with version 2 and are also __backward compatible__ with version 1. +- Docker Compose files in this repository work with version 2 and are also __backward compatible__ with version 1. - If you are using the distribution installation the `docker compose` command used below has to be changed to `docker-compose` instead. -## VLSD Image +## VLS standalone Setup -``` +Below are the steps required to run `vlsd2` as a standalone container. + +### Docker Image + +```bash +# Pull the latest release image for vlsd +docker pull registry.gitlab.com/lightning-signer/vls-container/vlsd:latest +docker tag registry.gitlab.com/lightning-signer/vls-container/vlsd:latest vlsd:latest +# Build the latest docker image cd vlsd docker build -t vlsd . ``` -## Volume Creation +### Volume Creation +```bash +docker volume create vls_data ``` + +### Environment Variables + +VLS container needs the follwing environment variables set: +- `BITCOIND_RPC_URL`: URL of `bitcoind`'s RPC port. +- `VLS_NETWORK`: `testnet` or `regtest`. + +Frequently used optional environment variables are: +- `VLS_PERMISSIVE` +- `RUST_LOG` + +### `vlsd2` Command Arguments + +Required command arguments: +- `connect`: URL of `remote_hsmd_socket` running in the lightning node. + +For information on all possible arguments to `vlsd` see [documentation](https://gitlab.com/lightning-signer/validating-lightning-signer/-/blob/main/vls-proxy/src/config.rs?ref_type=heads). + +### Running container + +There is a `docker-compose.yml` in the `vlsd` folder which can be used to run a standalone `vlsd` service with `network_mode` set to host. + +```bash +cd vlsd +docker compose up +``` + +**_Note_**: Make sure to set `BITCOIND_RPC_URL` and `CLN_RMEOTE_HSMD_URL` as either environment variables or in the `docker-compose.yml` file before running the above command. + +If you wish to run it as a standalone container without using `docker-compose` you can use the following command: + +```bash +docker run \ + -d \ + --rm \ + --name vlsd \ + --network host \ + -e VLS_NETWORK=testnet \ + -e BITCOIND_RPC_URL=$BITCOIND_RPC_URL \ + --mount 'type=volume,src=vls_data,dst=/home/vls/.lightning-signer' \ + vlsd \ + --connect=$CLN_REMOTE_HSMD_URL +``` + +## Single Node Setup + +You can run `bitcoind`, `lightningd`, `txood` and `vlsd` on a single node using available docker compose file in the main directory. + +**_Note_**: Use this only for experimentation and testing purposes as running `vlsd` on the same machine as `CLN` is not as secure as running it on dedicated hardware. + +### Volume Creation + +```bash docker volume create bitcoin_data docker volume create lightning_data docker volume create txoo_data docker volume create vls_data ``` -## Docker Compose Run +### Docker Compose Run -``` +```bash docker compose up --build ``` -## Using Bitcoin Chains +### Selecting Bitcoin Chains -We have three possible overrides over the default `testnet` configuration in `docker-compose.yml`: +We have two possible overrides over the default `testnet` configuration in `docker-compose.yml`: - `docker-compose.testnet.yml` - `docker-compose.regtest.yml` To use override we have to pass it down both the config using `-f` flag: -``` -export COMPOSE_PROJECT_NAME= -docker compose -f docker-compose.yml -f up --build +```bash +export DOCKER_COMPOSE_OVERRIDE=docker-compose.testnet.yml +export COMPOSE_PROJECT_NAME=testnet +docker compose -f docker-compose.yml -f $DOCKER_COMPOSE_OVERRIDE up --build ``` __Note__: Even while using `testnet` running using the override is recommended as that will expose the `P2P` port for `bitcoind` and `P2P` port for `lightningd` on the host. -## Additional Regtest Commands +### Additional Regtest Commands We have to run these commands after bitcoind is up and running. Create Wallet: -``` +```bash docker container exec bitcoind-regtest bitcoin-cli createwallet default ``` Generate Address for node: -``` +```bash docker container exec bitcoind-regtest bitcoin-cli getnewaddress ``` Generate Blocks -``` +```bash docker container exec bitcoind-regtest bitcoin-cli generatetoaddress 50 $NODE_ADDRESS ``` -## Testnet CLN + CLBOSS Commands +### Testnet CLN + CLBOSS Commands -``` +```bash # Create a connection to a random node (maybe from 1ML.com) docker container exec lightningd-test lightning-cli --testnet connect \ 02ae1e6091d2a9c4db5096558668d2456b1c0e9067cb72273eab1199bcfb208888 67.227.190.47:9735 @@ -105,7 +169,17 @@ docker container exec lightningd-test lightning-cli --testnet clboss-status | le docker container exec lightningd-test lightning-cli --testnet summary ``` -### References +## Future Work + +- [x] standalone dockerfile for vlsd +- [x] docker compose for vls +- [ ] standalone docker image for `lightning-storage-server` +- [ ] standalone docker image for `txood` +- [ ] standalone docker image for `lightningd` +- [ ] standalone docker image for `bitcoind` +- [ ] supporting signet + +## References - [bitcoind](https://github.com/ruimarinho/docker-bitcoin-core/blob/master/23/alpine/Dockerfile) by @ruimarinho - [lightningd with clboss](https://github.com/tsjk/docker-core-lightning/blob/main/Dockerfile) by @tsjk From a851c5c0d4afc25466495fd6f6a07a22ce48dfac Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Wed, 15 Nov 2023 08:48:29 +0530 Subject: [PATCH 4/5] dockerfile: all services standalone - bitcoind - lightnind - txood - vlsd: already - update future work Signed-off-by: Lakshya Singh --- README.md | 12 ++++++++---- bitcoind/Dockerfile | 6 +++--- {assets => bitcoind}/bitcoin.conf | 0 docker-compose.yml | 9 +++------ lightningd/Dockerfile | 12 ++++++------ lightningd/bitcoin.conf | 1 + {assets => lightningd}/regtest-config | 0 {assets => lightningd}/testnet-config | 0 txood/Dockerfile | 2 +- 9 files changed, 22 insertions(+), 20 deletions(-) rename {assets => bitcoind}/bitcoin.conf (100%) create mode 120000 lightningd/bitcoin.conf rename {assets => lightningd}/regtest-config (100%) rename {assets => lightningd}/testnet-config (100%) diff --git a/README.md b/README.md index 350cc91..404ddcc 100644 --- a/README.md +++ b/README.md @@ -173,11 +173,15 @@ docker container exec lightningd-test lightning-cli --testnet summary - [x] standalone dockerfile for vlsd - [x] docker compose for vls -- [ ] standalone docker image for `lightning-storage-server` -- [ ] standalone docker image for `txood` -- [ ] standalone docker image for `lightningd` -- [ ] standalone docker image for `bitcoind` +- [x] standalone docker image for `txood` +- [x] standalone docker image for `lightningd` +- [x] standalone docker image for `bitcoind` - [ ] supporting signet +- [ ] Profile configuration to run `vls` +- [ ] Healthcheck for `txoo` +- [ ] Healthcheck for `vls` +- [ ] Lightning Storage Server Dockerfile and Compose Service +- [ ] Postgres Service for Lightning Storage Server ## References diff --git a/bitcoind/Dockerfile b/bitcoind/Dockerfile index 1dbc75c..ed23efd 100644 --- a/bitcoind/Dockerfile +++ b/bitcoind/Dockerfile @@ -114,12 +114,12 @@ COPY --from=bitcoin-core ${BITCOIN_PREFIX}/bin/bitcoin-cli /usr/bin/bitcoin-cli COPY --from=bitcoin-core ${BITCOIN_PREFIX}/bin/bitcoin-tx /usr/bin/bitcoin-tx COPY --from=bitcoin-core ${BITCOIN_PREFIX}/bin/bitcoind /usr/bin/bitcoind -COPY bitcoind/entrypoint.sh /entrypoint.sh +COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh -COPY assets/bitcoin.conf /bitcoin.conf +COPY bitcoin.conf /bitcoin.conf -COPY bitcoind/healthcheck.sh /healthcheck.sh +COPY healthcheck.sh /healthcheck.sh RUN chmod +x /healthcheck.sh RUN mkdir "${BITCOIN_DATA}" diff --git a/assets/bitcoin.conf b/bitcoind/bitcoin.conf similarity index 100% rename from assets/bitcoin.conf rename to bitcoind/bitcoin.conf diff --git a/docker-compose.yml b/docker-compose.yml index c55e8fc..7c2242b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,7 @@ version: "3.8" services: bitcoin-core: build: - dockerfile: ./bitcoind/Dockerfile - context: . + context: ./bitcoind image: bitcoind container_name: bitcoind-test volumes: @@ -19,8 +18,7 @@ services: core-lightning: build: - dockerfile: ./lightningd/Dockerfile - context: . + context: ./lightningd image: lightningd container_name: lightningd-test command: @@ -46,8 +44,7 @@ services: txoo: build: - dockerfile: ./txood/Dockerfile - context: . + context: ./txood image: txood container_name: txood-test restart: unless-stopped diff --git a/lightningd/Dockerfile b/lightningd/Dockerfile index c3f040b..879ca72 100644 --- a/lightningd/Dockerfile +++ b/lightningd/Dockerfile @@ -118,12 +118,12 @@ RUN addgroup -S lightning && adduser -S lightning -G lightning && \ mkdir -p ${LIGHTNINGD_DATA} && \ chown -R lightning:lightning ${LIGHTNINGD_DATA} -COPY assets/bitcoin.conf "${BITCOIND_HOME}"/bitcoin.conf +COPY bitcoin.conf "${BITCOIND_HOME}"/bitcoin.conf -COPY lightningd/entrypoint.sh /entrypoint.sh +COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh -COPY lightningd/healthcheck.sh /healthcheck.sh +COPY healthcheck.sh /healthcheck.sh RUN chmod +x /healthcheck.sh VOLUME ["${LIGHTNINGD_DATA}"] @@ -131,9 +131,9 @@ VOLUME ["${LIGHTNINGD_DATA}"] RUN mkdir -p "${BITCOIND_HOME}" && \ chown -R lightning:lightning "${BITCOIND_HOME}" -COPY assets/bitcoin.conf "${BITCOIND_HOME}"/bitcoin.conf -COPY assets/testnet-config /testnet-config -COPY assets/regtest-config /regtest-config +COPY bitcoin.conf "${BITCOIND_HOME}"/bitcoin.conf +COPY testnet-config /testnet-config +COPY regtest-config /regtest-config USER lightning diff --git a/lightningd/bitcoin.conf b/lightningd/bitcoin.conf new file mode 120000 index 0000000..b1895bf --- /dev/null +++ b/lightningd/bitcoin.conf @@ -0,0 +1 @@ +../bitcoind/bitcoin.conf \ No newline at end of file diff --git a/assets/regtest-config b/lightningd/regtest-config similarity index 100% rename from assets/regtest-config rename to lightningd/regtest-config diff --git a/assets/testnet-config b/lightningd/testnet-config similarity index 100% rename from assets/testnet-config rename to lightningd/testnet-config diff --git a/txood/Dockerfile b/txood/Dockerfile index 5a1b18b..9611794 100644 --- a/txood/Dockerfile +++ b/txood/Dockerfile @@ -32,7 +32,7 @@ RUN apk update && \ COPY --from=builder /build/txoo/target/release/txood /usr/bin/txood -COPY txood/entrypoint.sh /entrypoint.sh +COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh VOLUME ["/root/.txoo/"] From cdd28817e235403a9ae8c8de9f147dd7d6d8a0bf Mon Sep 17 00:00:00 2001 From: Lakshya Singh Date: Sun, 19 Nov 2023 01:24:27 +0530 Subject: [PATCH 5/5] compose: enable profile vls for vls service - allow running all services without vls on same node Signed-off-by: Lakshya Singh --- README.md | 14 ++++++++++++-- docker-compose.yml | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 404ddcc..7bea844 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ docker volume create vls_data ### Docker Compose Run ```bash -docker compose up --build +docker compose --profile vls up --build ``` ### Selecting Bitcoin Chains @@ -125,11 +125,21 @@ To use override we have to pass it down both the config using `-f` flag: ```bash export DOCKER_COMPOSE_OVERRIDE=docker-compose.testnet.yml export COMPOSE_PROJECT_NAME=testnet -docker compose -f docker-compose.yml -f $DOCKER_COMPOSE_OVERRIDE up --build +docker compose --profile vls -f docker-compose.yml -f $DOCKER_COMPOSE_OVERRIDE up --build ``` __Note__: Even while using `testnet` running using the override is recommended as that will expose the `P2P` port for `bitcoind` and `P2P` port for `lightningd` on the host. +### Single Node without VLS + +To run a single node without `vls` service we can use the same `docker-compose.yml` file in the main directory by just removing the `profile` flag `vls` from all commands. + +```bash +docker compose up --build +``` + +Above command will run `bitcoind`, `lightningd` and `txood` services on a single node. + ### Additional Regtest Commands We have to run these commands after bitcoind is up and running. diff --git a/docker-compose.yml b/docker-compose.yml index 7c2242b..ee3182a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,6 +66,8 @@ services: context: ./vlsd image: vlsd container_name: vlsd-test + profiles: + - vls command: - --log-level=info - --connect=http://core-lightning:7701