subscribe_block_headers - Connection reset by peer

[scottyeager]

I've seen this error a couple times now on one system running the ingester (but only one... isn't that odd?).

The subscribe_block_headers thread is somewhat of a special case, since it's an imported function. I guess the thing to do is wrap it in a function with an infinite loop and a try block. Or check that it's alive from the main thread and respawn as needed.

Traceback (most recent call last):
  File "/root/node-status-bot/ingester.py", line 373, in <module>
Exception in thread Thread-3 (subscribe_block_headers):
Traceback (most recent call last):
  File "/usr/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
    block = client.sub.get_block(block_number=last_block)    self.run()
  File "/usr/local/lib/python3.10/dist-packages/substrateinterface/base.py", line 2463, in get_block

  File "/usr/lib/python3.10/threading.py", line 953, in run
    block_hash = self.get_block_hash(block_number)    self._target(*self._args, **self._kwargs)
  File "/usr/local/lib/python3.10/dist-packages/substrateinterface/base.py", line 479, in get_block_hash

  File "/usr/local/lib/python3.10/dist-packages/substrateinterface/base.py", line 2572, in subscribe_block_headers
    return self.__get_block_handler(    response = self.rpc_request("chain_getBlockHash", [block_id])
  File "/usr/local/lib/python3.10/dist-packages/substrateinterface/base.py", line 2425, in __get_block_handler

  File "/usr/local/lib/python3.10/dist-packages/substrateinterface/base.py", line 329, in rpc_request
    self.__rpc_message_queue.append(json.loads(self.websocket.recv()))
  File "/usr/local/lib/python3.10/dist-packages/websocket/_core.py", line 388, in recv
    opcode, data = self.recv_data()
  File "/usr/local/lib/python3.10/dist-packages/websocket/_core.py", line 416, in recv_data
    result = self.rpc_request(f"chain_subscribe{rpc_method_prefix}Heads", [], result_handler=result_handler)
  File "/usr/local/lib/python3.10/dist-packages/substrateinterface/base.py", line 329, in rpc_request
    opcode, frame = self.recv_data_frame(control_frame)
  File "/usr/local/lib/python3.10/dist-packages/websocket/_core.py", line 437, in recv_data_frame
    self.__rpc_message_queue.append(json.loads(self.websocket.recv()))
    frame = self.recv_frame()  File "/usr/local/lib/python3.10/dist-packages/websocket/_core.py", line 388, in recv

  File "/usr/local/lib/python3.10/dist-packages/websocket/_core.py", line 478, in recv_frame
    return self.frame_buffer.recv_frame()
    opcode, data = self.recv_data()  File "/usr/local/lib/python3.10/dist-packages/websocket/_abnf.py", line 363, in recv_frame

  File "/usr/local/lib/python3.10/dist-packages/websocket/_core.py", line 416, in recv_data
    self.recv_header()
  File "/usr/local/lib/python3.10/dist-packages/websocket/_abnf.py", line 319, in recv_header
    opcode, frame = self.recv_data_frame(control_frame)
  File "/usr/local/lib/python3.10/dist-packages/websocket/_core.py", line 437, in recv_data_frame
    header = self.recv_strict(2)
  File "/usr/local/lib/python3.10/dist-packages/websocket/_abnf.py", line 398, in recv_strict
    frame = self.recv_frame()
  File "/usr/local/lib/python3.10/dist-packages/websocket/_core.py", line 478, in recv_frame
    bytes_ = self.recv(min(16384, shortage))
  File "/usr/local/lib/python3.10/dist-packages/websocket/_core.py", line 563, in _recv
    return self.frame_buffer.recv_frame()
    return recv(self.sock, bufsize)  File "/usr/local/lib/python3.10/dist-packages/websocket/_abnf.py", line 363, in recv_frame

  File "/usr/local/lib/python3.10/dist-packages/websocket/_socket.py", line 90, in recv
    raise WebSocketConnectionClosedException("socket is already closed.")
    self.recv_header()websocket._exceptions.
  File "/usr/local/lib/python3.10/dist-packages/websocket/_abnf.py", line 319, in recv_header
WebSocketConnectionClosedException: socket is already closed.
    header = self.recv_strict(2)
  File "/usr/local/lib/python3.10/dist-packages/websocket/_abnf.py", line 398, in recv_strict
    bytes_ = self.recv(min(16384, shortage))
  File "/usr/local/lib/python3.10/dist-packages/websocket/_core.py", line 563, in _recv
    return recv(self.sock, bufsize)
  File "/usr/local/lib/python3.10/dist-packages/websocket/_socket.py", line 129, in recv
    raise WebSocketConnectionClosedException("Connection to remote host was lost.")
websocket._exceptions.WebSocketConnectionClosedException: Connection to remote host was lost.
----------------------------------------
Exception occurred during processing of request from ('167.94.145.108', 50394)
Traceback (most recent call last):
  File "/usr/lib/python3.10/socketserver.py", line 683, in process_request_thread
    self.finish_request(request, client_address)
  File "/usr/lib/python3.10/socketserver.py", line 360, in finish_request
    self.RequestHandlerClass(request, client_address, self)
  File "/usr/lib/python3.10/socketserver.py", line 747, in __init__
    self.handle()
  File "/usr/lib/python3.10/wsgiref/simple_server.py", line 118, in handle
    self.raw_requestline = self.rfile.readline(65537)
  File "/usr/lib/python3.10/socket.py", line 705, in readinto
    return self._sock.recv_into(b)
ConnectionResetError: [Errno 104] Connection reset by peer
----------------------------------------
Exception occurred during processing of request from ('104.248.253.248', 48952)
Traceback (most recent call last):
  File "/usr/lib/python3.10/socketserver.py", line 683, in process_request_thread
    self.finish_request(request, client_address)
  File "/usr/lib/python3.10/socketserver.py", line 360, in finish_request
    self.RequestHandlerClass(request, client_address, self)
  File "/usr/lib/python3.10/socketserver.py", line 747, in __init__
    self.handle()
  File "/usr/lib/python3.10/wsgiref/simple_server.py", line 118, in handle
    self.raw_requestline = self.rfile.readline(65537)
  File "/usr/lib/python3.10/socket.py", line 705, in readinto
    return self._sock.recv_into(b)
ConnectionResetError: [Errno 104] Connection reset by peer
----------------------------------------
----------------------------------------
Exception occurred during processing of request from ('80.66.83.215', 48997)
Traceback (most recent call last):
  File "/usr/lib/python3.10/socketserver.py", line 683, in process_request_thread
    self.finish_request(request, client_address)
  File "/usr/lib/python3.10/socketserver.py", line 360, in finish_request
    self.RequestHandlerClass(request, client_address, self)
  File "/usr/lib/python3.10/socketserver.py", line 747, in __init__
    self.handle()
  File "/usr/lib/python3.10/wsgiref/simple_server.py", line 118, in handle
    self.raw_requestline = self.rfile.readline(65537)
  File "/usr/lib/python3.10/socket.py", line 705, in readinto
    return self._sock.recv_into(b)
ConnectionResetError: [Errno 104] Connection reset by peer
----------------------------------------