Merge pull request #23 from BlueWallet/unlock-payments-script

ADD: unlock stuck payments script

class/Paym.js

@@ -2,23 +2,128 @@ var crypto = require('crypto');
 var lightningPayReq = require('bolt11');
 import { BigNumber } from 'bignumber.js';
 
-export class Payment {
+export class Paym {
   constructor(redis, bitcoindrpc, lightning) {
     this._redis = redis;
     this._bitcoindrpc = bitcoindrpc;
     this._lightning = lightning;
     this._decoded = false;
+    this._bolt11 = false;
+    this._isPaid = null;
+  }
+
+  setInvoice(bolt11) {
+    this._bolt11 = bolt11;
   }
 
   async decodePayReqViaRpc(invoice) {
+    let that = this;
     return new Promise(function(resolve, reject) {
-      this._lightning.decodePayReq({ pay_req: invoice }, function(err, info) {
+      that._lightning.decodePayReq({ pay_req: invoice }, function(err, info) {
         if (err) return reject(err);
+        that._decoded = info;
         return resolve(info);
       });
     });
   }
 
+  async queryRoutes() {
+    if (!this._bolt11) throw new Error('bolt11 is not provided');
+    if (!this._decoded) await this.decodePayReqViaRpc(this._bolt11);
+
+    var request = {
+      pub_key: this._decoded.destination,
+      amt: this._decoded.num_satoshis,
+      num_routes: 1,
+      final_cltv_delta: 144,
+      fee_limit: { fixed: Math.floor(this._decoded.num_satoshis * 0.01) },
+    };
+    let that = this;
+    return new Promise(function(resolve, reject) {
+      that._lightning.queryRoutes(request, function(err, response) {
+        if (err) return reject(err);
+        resolve(response);
+      });
+    });
+  }
+
+  async sendToRouteSync(routes) {
+    if (!this._bolt11) throw new Error('bolt11 is not provided');
+    if (!this._decoded) await this.decodePayReqViaRpc(this._bolt11);
+
+    let request = {
+      payment_hash_string: this._decoded.payment_hash,
+      routes: routes,
+    };
+
+    let that = this;
+    return new Promise(function(resolve, reject) {
+      that._lightning.sendToRouteSync(request, function(err, response) {
+        if (err) reject(err);
+        resolve(that.processSendPaymentResponse(response));
+      });
+    });
+  }
+
+  processSendPaymentResponse(payment) {
+    if (payment && payment.payment_route && payment.payment_route.total_amt_msat) {
+      // paid just now
+      this._isPaid = true;
+      payment.payment_route.total_fees = +payment.payment_route.total_fees + Math.floor(+payment.payment_route.total_amt * 0.01);
+      if (this._bolt11) payment.pay_req = this._bolt11;
+      if (this._decoded) payment.decoded = this._decoded;
+    }
+
+    if (payment.payment_error && payment.payment_error.indexOf('already paid') !== -1) {
+      // already paid
+      this._isPaid = true;
+      if (this._decoded) {
+        payment.decoded = this._decoded;
+        if (this._bolt11) payment.pay_req = this._bolt11;
+        // trying to guess the fee
+        payment.payment_route = payment.payment_route || {};
+        payment.payment_route.total_fees = Math.floor(this._decoded.num_satoshis * 0.01);
+        payment.payment_route.total_amt = this._decoded.num_satoshis;
+      }
+    }
+
+    if (payment.payment_error && payment.payment_error.indexOf('unable to') !== -1) {
+      // failed to pay
+      this._isPaid = false;
+    }
+
+    if (payment.payment_error && payment.payment_error.indexOf('FinalExpiryTooSoon') !== -1) {
+      this._isPaid = false;
+    }
+
+    if (payment.payment_error && payment.payment_error.indexOf('payment is in transition') !== -1) {
+      this._isPaid = null; // null is default, but lets set it anyway
+    }
+
+    return payment;
+  }
+
+  /**
+   * Returns NULL if unknown, true if its paid, false if its unpaid
+   * (judging by error in sendPayment response)
+   *
+   * @returns {boolean|null}
+   */
+  getIsPaid() {
+    return this._isPaid;
+  }
+
+  async attemptPayToRoute() {
+    let routes = await this.queryRoutes();
+    return await this.sendToRouteSync(routes.routes);
+  }
+
+  async isExpired() {
+    if (!this._bolt11) throw new Error('bolt11 is not provided');
+    const decoded = await this.decodePayReqViaRpc(this._bolt11);
+    return +decoded.timestamp + +decoded.expiry < +new Date() / 1000;
+  }
+
   decodePayReq(payReq) {
     this._decoded = lightningPayReq.decode(payReq);
   }

class/User.js

@@ -453,4 +453,20 @@ export class User {
       .digest()
       .toString('hex');
   }
+
+  /**
+   * Shuffles array in place. ES6 version
+   * @param {Array} a items An array containing the items.
+   */
+  static _shuffle(a) {
+    for (let i = a.length - 1; i > 0; i--) {
+      const j = Math.floor(Math.random() * (i + 1));
+      [a[i], a[j]] = [a[j], a[i]];
+    }
+    return a;
+  }
+
+  static async _sleep(s) {
+    return new Promise(r => setTimeout(r, s * 1000));
+  }
 }

class/index.js

@@ -1,2 +1,3 @@
 export * from './User';
 export * from './Lock';
+export * from './Paym';

controllers/api.js

@@ -187,7 +187,8 @@ router.post('/payinvoice', async function(req, res) {
         // payment callback
         await u.unlockFunds(req.body.invoice);
         if (payment && payment.payment_route && payment.payment_route.total_amt_msat) {
-          payment.payment_route.total_fees = +payment.payment_route.total_fees + Math.floor(+payment.payment_route.total_amt * 0.01);
+          let PaymentShallow = new Paym(false, false, false);
+          payment = PaymentShallow.processSendPaymentResponse(payment);
           userBalance -= +payment.payment_route.total_fees + +payment.payment_route.total_amt;
           u.saveBalance(userBalance);
           payment.pay_req = req.body.invoice;

rpc.proto

@@ -3,6 +3,9 @@ syntax = "proto3";
 // import "google/api/annotations.proto";
 
 package lnrpc;
+
+option go_package = "github.com/lightningnetwork/lnd/lnrpc";
+
 /**
  * Comments in this file will be directly parsed into the API
  * Documentation as descriptions of the associated method, message, or field.
@@ -231,6 +234,16 @@ service Lightning {
         };
     }
 
+    /** lncli: `listunspent`
+    ListUnspent returns a list of all utxos spendable by the wallet with a
+	number of confirmations between the specified minimum and maximum.
+    */
+    rpc ListUnspent (ListUnspentRequest) returns (ListUnspentResponse) {
+        option (google.api.http) = {
+            get: "/v1/utxos"
+        };
+    }
+
     /**
     SubscribeTransactions creates a uni-directional stream from the server to
     the client in which any newly discovered transactions relevant to the
@@ -260,7 +273,12 @@ service Lightning {
     signature string is `zbase32` encoded and pubkey recoverable, meaning that
     only the message digest and signature are needed for verification.
     */
-    rpc SignMessage (SignMessageRequest) returns (SignMessageResponse);
+    rpc SignMessage (SignMessageRequest) returns (SignMessageResponse) {
+        option (google.api.http) = {
+            post: "/v1/signmessage"
+            body: "*"
+        };
+    }
 
     /** lncli: `verifymessage`
     VerifyMessage verifies a signature over a msg. The signature must be
@@ -268,7 +286,12 @@ service Lightning {
     channel database. In addition to returning the validity of the signature,
     VerifyMessage also returns the recovered pubkey from the signature.
     */
-    rpc VerifyMessage (VerifyMessageRequest) returns (VerifyMessageResponse);
+    rpc VerifyMessage (VerifyMessageRequest) returns (VerifyMessageResponse) {
+        option (google.api.http) = {
+            post: "/v1/verifymessage"
+            body: "*"
+        };
+    }
 
     /** lncli: `connect`
     ConnectPeer attempts to establish a connection to a remote peer. This is at
@@ -336,6 +359,14 @@ service Lightning {
         };
     }
 
+    /** lncli: `subscribechannelevents`
+    SubscribeChannelEvents creates a uni-directional stream from the server to
+    the client in which any updates relevant to the state of the channels are
+    sent over. Events include new active channels, inactive channels, and closed
+    channels.
+    */
+    rpc SubscribeChannelEvents (ChannelEventSubscription) returns (stream ChannelEventUpdate);
+
     /** lncli: `closedchannels`
     ClosedChannels returns a description of all the closed channels that 
     this node was a participant in.
@@ -455,10 +486,8 @@ service Lightning {
     paginated responses, allowing users to query for specific invoices through
     their add_index. This can be done by using either the first_index_offset or
     last_index_offset fields included in the response as the index_offset of the
-    next request. The reversed flag is set by default in order to paginate
+    next request. By default, the first 100 invoices created will be returned.
-    backwards. If you wish to paginate forwards, you must explicitly set the
+    Backwards pagination is also supported through the Reversed flag.
-    flag to false. If none of the parameters are specified, then the last 100
-    invoices will be returned.
     */
     rpc ListInvoices (ListInvoiceRequest) returns (ListInvoiceResponse) {
         option (google.api.http) = {
@@ -647,6 +676,26 @@ service Lightning {
     };
 }
 
+message Utxo {
+    /// The type of address
+    AddressType type = 1 [json_name = "address_type"];
+
+    /// The address
+    string address = 2 [json_name = "address"];
+
+    /// The value of the unspent coin in satoshis
+    int64 amount_sat = 3 [json_name = "amount_sat"];
+
+    /// The pkscript in hex
+    string pk_script = 4 [json_name = "pk_script"];
+
+    /// The outpoint in format txid:n
+    OutPoint outpoint = 5 [json_name = "outpoint"];
+
+    /// The number of confirmations for the Utxo
+    int64 confirmations = 6 [json_name = "confirmations"];
+}
+
 message Transaction {
     /// The transaction hash
     string tx_hash = 1 [ json_name = "tx_hash" ];
@@ -725,11 +774,18 @@ message SendRequest {
     send the payment.
     */
     FeeLimit fee_limit = 8;
+
+    /**
+    The channel id of the channel that must be taken to the first hop. If zero,
+    any channel may be used.
+    */
+    uint64 outgoing_chan_id = 9;
 }
 message SendResponse {
     string payment_error = 1 [json_name = "payment_error"];
     bytes payment_preimage = 2 [json_name = "payment_preimage"];
     Route payment_route = 3 [json_name = "payment_route"];
+    bytes payment_hash = 4 [json_name = "payment_hash"];
 }
 
 message SendToRouteRequest {
@@ -739,8 +795,16 @@ message SendToRouteRequest {
     /// An optional hex-encoded payment hash to be used for the HTLC.
     string payment_hash_string = 2;
 
-    /// The set of routes that should be used to attempt to complete the payment.
+    /**
-    repeated Route routes = 3;
+    Deprecated. The set of routes that should be used to attempt to complete the
+    payment. The possibility to pass in multiple routes is deprecated and 
+    instead the single route field below should be used in combination with the 
+    streaming variant of SendToRoute.
+    */
+    repeated Route routes = 3 [deprecated = true];
+
+    /// Route that should be used to attempt to complete the payment.
+    Route route = 4;
 }
 
 message ChannelPoint {
@@ -756,6 +820,17 @@ message ChannelPoint {
     uint32 output_index = 3 [json_name = "output_index"];
 }
 
+message OutPoint {
+    /// Raw bytes representing the transaction id.
+    bytes txid_bytes = 1 [json_name = "txid_bytes"];
+
+    /// Reversed, hex-encoded string representing the transaction id.
+    string txid_str = 2 [json_name = "txid_str"];
+
+    /// The index of the output on the transaction.
+    uint32 output_index = 3 [json_name = "output_index"];
+}
+
 message LightningAddress {
     /// The identity pubkey of the Lightning node
     string pubkey = 1 [json_name = "pubkey"];
@@ -791,24 +866,45 @@ message SendCoinsRequest {
 
     /// A manual fee rate set in sat/byte that should be used when crafting the transaction.
     int64 sat_per_byte = 5;
+
+    /**
+    If set, then the amount field will be ignored, and lnd will attempt to
+    send all the coins under control of the internal wallet to the specified
+    address.
+    */
+    bool send_all = 6; 
 }
 message SendCoinsResponse {
     /// The transaction ID of the transaction
     string txid = 1 [json_name = "txid"];
 }
 
+message ListUnspentRequest {
+    /// The minimum number of confirmations to be included.
+    int32 min_confs = 1;
+
+    /// The maximum number of confirmations to be included.
+    int32 max_confs = 2;
+}
+message ListUnspentResponse {
+    /// A list of utxos
+    repeated Utxo utxos = 1 [json_name = "utxos"];
+}
+
 /** 
 `AddressType` has to be one of:
 
 - `p2wkh`: Pay to witness key hash (`WITNESS_PUBKEY_HASH` = 0)
 - `np2wkh`: Pay to nested witness key hash (`NESTED_PUBKEY_HASH` = 1)
 */
-message NewAddressRequest {
+enum AddressType {
-    enum AddressType {
         WITNESS_PUBKEY_HASH = 0;
         NESTED_PUBKEY_HASH = 1;
-    }
+        UNUSED_WITNESS_PUBKEY_HASH = 2;
+        UNUSED_NESTED_PUBKEY_HASH = 3;
+}
 
+message NewAddressRequest {
     /// The address type
     AddressType type = 1;
 }
@@ -944,8 +1040,11 @@ message Channel {
     */
     uint32 csv_delay = 16 [json_name = "csv_delay"];
 
-    /// Whether this channel is advertised to the network or not
+    /// Whether this channel is advertised to the network or not.
     bool private = 17 [json_name = "private"];
+
+    /// True if we were the ones that created the channel.
+    bool initiator = 18 [json_name = "initiator"];
 }
 
 
@@ -1075,11 +1174,13 @@ message GetInfoResponse {
     /// Whether the wallet's view is synced to the main chain
     bool synced_to_chain = 9 [json_name = "synced_to_chain"];
 
-    /// Whether the current node is connected to testnet
+    /** 
-    bool testnet = 10 [json_name = "testnet"];
+    Whether the current node is connected to testnet. This field is 
+    deprecated and the network field should be used instead 
+    **/
+    bool testnet = 10 [json_name = "testnet", deprecated = true];
 
-    /// A list of active chains the node is connected to
+    reserved 11;
-    repeated string chains = 11 [json_name = "chains"];
 
     /// The URIs of the current node.
     repeated string uris = 12 [json_name = "uris"];
@@ -1092,6 +1193,17 @@ message GetInfoResponse {
 
     /// Number of inactive channels
     uint32 num_inactive_channels = 15 [json_name = "num_inactive_channels"];
+
+    /// A list of active chains the node is connected to
+    repeated Chain chains = 16 [json_name = "chains"];
+}
+
+message Chain {
+    /// The blockchain the node is on (eg bitcoin, litecoin)
+    string chain = 1 [json_name = "chain"];
+
+    /// The network the node is on (eg regtest, testnet, mainnet)
+    string network = 2 [json_name = "network"];
 }
 
 message ConfirmationUpdate {
@@ -1132,7 +1244,6 @@ message CloseChannelRequest {
 message CloseStatusUpdate {
     oneof update {
         PendingUpdate close_pending = 1 [json_name = "close_pending"];
-        ConfirmationUpdate confirmation = 2 [json_name = "confirmation"];
         ChannelCloseUpdate chan_close = 3 [json_name = "chan_close"];
     }
 }
@@ -1179,7 +1290,6 @@ message OpenChannelRequest {
 message OpenStatusUpdate {
     oneof update {
         PendingUpdate chan_pending = 1 [json_name = "chan_pending"];
-        ConfirmationUpdate confirmation = 2 [json_name = "confirmation"];
         ChannelOpenUpdate chan_open = 3 [json_name = "chan_open"];
     }
 }
@@ -1306,6 +1416,27 @@ message PendingChannelsResponse {
     repeated WaitingCloseChannel waiting_close_channels = 5 [ json_name = "waiting_close_channels" ];
 }
 
+message ChannelEventSubscription {
+}
+
+message ChannelEventUpdate {
+    oneof channel {
+        Channel open_channel = 1 [ json_name = "open_channel" ];
+        ChannelCloseSummary closed_channel = 2 [ json_name = "closed_channel" ];
+        ChannelPoint active_channel = 3 [ json_name = "active_channel" ];
+        ChannelPoint inactive_channel = 4 [ json_name = "inactive_channel" ];
+    }
+
+    enum UpdateType {
+         OPEN_CHANNEL = 0;
+         CLOSED_CHANNEL = 1;
+         ACTIVE_CHANNEL = 2;
+         INACTIVE_CHANNEL = 3;
+    }
+
+    UpdateType type = 5 [ json_name = "type" ];
+}
+
 message WalletBalanceRequest {
 }
 message WalletBalanceResponse {
@@ -1468,6 +1599,7 @@ message RoutingPolicy {
     int64 fee_base_msat = 3 [json_name = "fee_base_msat"];
     int64 fee_rate_milli_msat = 4 [json_name = "fee_rate_milli_msat"];
     bool disabled = 5 [json_name = "disabled"];
+    uint64 max_htlc_msat = 6 [json_name = "max_htlc_msat"];
 }
 
 /**
@@ -1626,8 +1758,10 @@ message Invoice {
     */
     string memo = 1 [json_name = "memo"];
 
-    /// An optional cryptographic receipt of payment
+    /** Deprecated. An optional cryptographic receipt of payment which is not
-    bytes receipt = 2 [json_name = "receipt"];
+    implemented.
+    */
+    bytes receipt = 2 [json_name = "receipt", deprecated = true];
 
     /**
     The hex-encoded preimage (32 byte) which will allow settling an incoming
@@ -1642,7 +1776,7 @@ message Invoice {
     int64 value = 5 [json_name = "value"];
 
     /// Whether this invoice has been fulfilled
-    bool settled = 6 [json_name = "settled"];
+    bool settled = 6 [json_name = "settled", deprecated = true];
 
     /// When this invoice was created
     int64 creation_date = 7 [json_name = "creation_date"];
@@ -1720,7 +1854,19 @@ message Invoice {
     here as well.
     */
     int64 amt_paid_msat = 20 [json_name = "amt_paid_msat"];
+
+    enum InvoiceState {
+        OPEN = 0;
+        SETTLED = 1;
+        CANCELED = 2;
+    }
+
+    /**
+    The state the invoice is in.
+    */
+    InvoiceState state = 21 [json_name = "state"];
 }
+
 message AddInvoiceResponse {
     bytes r_hash = 1 [json_name = "r_hash"];
 
@@ -1953,15 +2099,18 @@ message ForwardingEvent {
     /// The outgoing channel ID that carried the preimage that completed the circuit.
     uint64 chan_id_out = 4 [json_name = "chan_id_out"];
 
-    /// The total amount of the incoming HTLC that created half the circuit.
+    /// The total amount (in satoshis) of the incoming HTLC that created half the circuit.
     uint64 amt_in = 5 [json_name = "amt_in"];
 
-    /// The total amount of the outgoign HTLC that created the second half of the circuit.
+    /// The total amount (in satoshis) of the outgoing HTLC that created the second half of the circuit.
     uint64 amt_out = 6 [json_name = "amt_out"];
 
-    /// The total fee that this payment circuit carried.
+    /// The total fee (in satoshis) that this payment circuit carried.
     uint64 fee = 7 [json_name = "fee"];
 
+    /// The total fee (in milli-satoshis) that this payment circuit carried.
+    uint64 fee_msat = 8 [json_name = "fee_msat"];
+
     // TODO(roasbeef): add settlement latency?
     //  * use FPE on the chan id?
     //  * also list failures?

scripts/process-locked-payments.js

@@ -0,0 +1,58 @@
+import { User, Lock, Paym } from '../class/';
+const config = require('../config');
+
+var Redis = require('ioredis');
+var redis = new Redis(config.redis);
+
+let bitcoinclient = require('../bitcoin');
+let lightning = require('../lightning');
+
+(async () => {
+  let keys = await redis.keys('locked_payments_for_*');
+  keys = User._shuffle(keys);
+
+  for (let key of keys) {
+    const userid = key.replace('locked_payments_for_', '');
+    console.log('===================================================================================');
+    console.log('userid=', userid);
+    let user = new User(redis, bitcoinclient, lightning);
+    user._userid = userid;
+    let lockedPayments = await user.getLockedPayments();
+
+    for (let lockedPayment of lockedPayments) {
+      console.log('processing lockedPayment=', lockedPayment);
+
+      let payment = new Paym(redis, bitcoinclient, lightning);
+      payment.setInvoice(lockedPayment.pay_req);
+      if (await payment.isExpired()) {
+        let sendResult;
+        try {
+          sendResult = await payment.attemptPayToRoute();
+        } catch (_) {
+          console.log(_);
+          await user.unlockFunds(lockedPayment.pay_req);
+          continue;
+        }
+        console.log('sendResult=', sendResult);
+        console.log('payment.getIsPaid() = ', payment.getIsPaid());
+        if (payment.getIsPaid() === true) {
+          console.log('paid successfully');
+          sendResult = payment.processSendPaymentResponse(sendResult); // adds fees
+          console.log('sendResult=', sendResult);
+          await user.savePaidLndInvoice(sendResult);
+          await user.unlockFunds(lockedPayment.pay_req);
+        } else if (payment.getIsPaid() === false) {
+          console.log('not paid, just evict the lock');
+          await user.unlockFunds(lockedPayment.pay_req);
+        } else {
+          console.log('payment is in unknown state');
+        }
+        console.log('sleeping 5 sec...');
+        console.log('-----------------------------------------------------------------------------------');
+        await User._sleep(5);
+      }
+    }
+  }
+  console.log('done');
+  process.exit();
+})();