Merge pull request #455 from yungwine/storage

Installation of segment archive node via TON Storage

Author: yungwine

modules/__init__.py

@@ -62,7 +62,8 @@ class Setting:
     'auto_backup': Setting('validator', None, 'Make validator backup every election'),
     'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'),
     'prometheus_url': Setting('prometheus', None, 'Prometheus pushgateway url'),
-    'onlyNode': Setting(None, None, 'MyTonCtrl will work only for collecting validator telemetry (if `sendTelemetry` is True), without participating in Elections and etc.')
+    'onlyNode': Setting(None, None, 'MyTonCtrl will work only for collecting validator telemetry (if `sendTelemetry` is True), without participating in Elections and etc.'),
+    'importGc': Setting(None, None, 'Delete imported archive blocks files. Restart mytoncore to apply this setting'),
 }
 
 

mytoncore/functions.py

@@ -53,8 +53,6 @@ def Event(local, event_name):
         EnableVcEvent(local)
     elif event_name == "validator down":
         ValidatorDownEvent(local)
-    elif event_name == "enable_ton_storage_provider":
-        enable_ton_storage_provider_event(local)
     elif event_name.startswith("enable_mode"):
         enable_mode(local, event_name)
     local.exit()
@@ -84,15 +82,6 @@ def ValidatorDownEvent(local):
 # end define
 
 
-def enable_ton_storage_provider_event(local):
-    config_path = local.db.ton_storage.provider.config_path
-    config = GetConfig(path=config_path)
-    key_bytes = base64.b64decode(config.ProviderKey)
-    ton = MyTonCore(local)
-    ton.import_wallet_with_version(key_bytes[:32], version="v3r2", wallet_name="provider_wallet_001")
-#end define
-
-
 def enable_mode(local, event_name):
     ton = MyTonCore(local)
     mode = event_name.split("_")[-1]
@@ -613,6 +602,35 @@ def check_initial_sync(local, ton):
         return
 
 
+def gc_import(local, ton):
+    if not ton.local.db.get('importGc', False):
+        return
+    local.add_log("GC import is running", "debug")
+    import_path = '/var/ton-work/db/import'
+    files = os.listdir(import_path)
+    if not files:
+        local.add_log("No files left to import", "debug")
+        ton.local.db['importGc'] = False
+        return
+    try:
+        status = ton.GetValidatorStatus()
+        node_seqno = int(status.shardclientmasterchainseqno)
+    except Exception as e:
+        local.add_log(f"Failed to get shardclientmasterchainseqno: {e}", "warning")
+        return
+    removed = 0
+    for file in files:
+        file_seqno = int(file.split('.')[1])
+        if node_seqno > file_seqno + 101:
+            try:
+                os.remove(os.path.join(import_path, file))
+                removed += 1
+            except PermissionError:
+                local.add_log(f"Failed to remove file {file}: Permission denied", "error")
+                continue
+    local.add_log(f"Removed {removed} import files up to {node_seqno} seqno", "debug")
+
+
 def General(local):
     local.add_log("start General function", "debug")
     ton = MyTonCore(local)
@@ -653,6 +671,9 @@ def General(local):
     if ton.in_initial_sync():
         local.start_cycle(check_initial_sync, sec=120, args=(local, ton))
 
+    if ton.local.db.get('importGc'):
+        local.start_cycle(gc_import, sec=300, args=(local, ton))
+
     thr_sleep()
 # end define
 

mytonctrl/mytonctrl.py

@@ -143,7 +143,6 @@ def inject_globals(func):
 		module.add_console_commands(console)
 
 	console.AddItem("benchmark", inject_globals(run_benchmark), local.translate("benchmark_cmd"))
-	# console.AddItem("activate_ton_storage_provider", inject_globals(activate_ton_storage_provider), local.translate("activate_ton_storage_provider_cmd"))
 
 	# Process input parameters
 	opts, args = getopt.getopt(argv,"hc:w:",["config=","wallets="])
@@ -176,23 +175,6 @@ def inject_globals(func):
 #end define
 
 
-def activate_ton_storage_provider(local, ton, args):
-	wallet_name = "provider_wallet_001"
-	wallet = ton.GetLocalWallet(wallet_name)
-	account = ton.GetAccount(wallet.addrB64)
-	if account.status == "active":
-		color_print("activate_ton_storage_provider - {green}Already activated{endc}")
-		#return
-	ton.ActivateWallet(wallet)
-	destination = "0:7777777777777777777777777777777777777777777777777777777777777777"
-	ton_storage = ton.GetSettings("ton_storage")
-	comment = f"tsp-{ton_storage.provider.pubkey}"
-	flags = ["-n", "-C", comment]
-	ton.MoveCoins(wallet, destination, 0.01, flags=flags)
-	color_print("activate_ton_storage_provider - {green}OK{endc}")
-#end define
-
-
 def about(local, ton, args):
 	from modules import get_mode, get_mode_settings
 	if len(args) != 1:

mytoninstaller/mytoninstaller.py

@@ -29,7 +29,6 @@
 	CreateSymlinks,
 	enable_ls_proxy,
 	enable_ton_storage,
-	enable_ton_storage_provider,
 	EnableMode, ConfigureFromBackup, ConfigureOnlyNode, SetInitialSync
 )
 from mytoninstaller.config import (
@@ -94,11 +93,13 @@ def Refresh(local):
 	ton_work_dir = "/var/ton-work/"
 	ton_bin_dir = bin_dir + "ton/"
 	ton_src_dir = src_dir + "ton/"
+	mtc_src_dir = src_dir + "mytonctrl/"
 	local.buffer.bin_dir = bin_dir
 	local.buffer.src_dir = src_dir
 	local.buffer.ton_work_dir = ton_work_dir
 	local.buffer.ton_bin_dir = ton_bin_dir
 	local.buffer.ton_src_dir = ton_src_dir
+	local.buffer.mtc_src_dir = mtc_src_dir
 	ton_db_dir = ton_work_dir + "db/"
 	keys_dir = ton_work_dir + "keys/"
 	local.buffer.ton_db_dir = ton_db_dir
@@ -162,7 +163,7 @@ def Enable(local, args):
 		print("'JR' - jsonrpc")
 		print("'THA' - ton-http-api")
 		print("'LSP' - ls-proxy")
-		print("'TSP' - ton-storage + ton-storage-provider")
+		print("'TS' - ton-storage")
 		print("Example: 'enable FN'")
 		return
 	if name == "THA":
@@ -227,9 +228,8 @@ def Event(local, name):
 		enable_ton_http_api(local)
 	if name == "enableLSP":
 		enable_ls_proxy(local)
-	if name == "enableTSP":
+	if name == "enableTS":
 		enable_ton_storage(local)
-		enable_ton_storage_provider(local)
 	if name == "clc":
 		ix = sys.argv.index("-i")
 		initBlock_b64 = sys.argv[ix+1]

mytoninstaller/scripts/process_hardforks.py

@@ -0,0 +1,101 @@
+import argparse
+import json
+import subprocess
+import sys
+import time
+import logging
+
+
+def run_vc(cmd: str):
+    if binary:
+        return subprocess.run([binary, '-k', client, '-p', server, '-a', address, '--cmd', cmd],
+                              stdout=subprocess.PIPE).stdout.decode()
+    else: # use symlink
+        return subprocess.run(['bash', 'validator-console', '--cmd', cmd],
+                              stdout=subprocess.PIPE).stdout.decode()
+
+def get_last_mc_seqno():
+    stats = run_vc('getstats')
+    for line in stats.split('\n'):
+        if line.startswith('masterchainblock'):
+            return int(line.split()[1].split(',')[2].split(')')[0])
+
+def restart_node():
+    return subprocess.run(['systemctl', 'restart', 'validator']).returncode
+
+def get_config():
+    with open(config_path) as f:
+        return json.load(f)
+
+def set_config(config: dict):
+    with open(config_path, 'w') as f:
+        f.write(json.dumps(config, indent=4))
+
+def set_hardforks(hfks: list):
+    c = get_config()
+    c['validator']['hardforks'] = hfks
+    set_config(c)
+
+def add_hardfork(hfk: dict):
+    c = get_config()
+    c['validator']['hardforks'].append(hfk)
+    set_config(c)
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(
+    level=logging.DEBUG,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+
+parser = argparse.ArgumentParser(description="Python script to process hardforks when syncing archive node.")
+parser.add_argument('--from-seqno', type=int, required=True, help='Global config path')
+parser.add_argument('--config-path', type=str, required=True, help='Global config path')
+parser.add_argument('--bin', type=str, help='VC bin')
+parser.add_argument('--client', type=str, help='VC Client')
+parser.add_argument('--server', type=str, help='VC Server')
+parser.add_argument('--address', type=str, help='VC Address')
+
+args = parser.parse_args()
+config_path = args.config_path
+binary = args.bin
+client = args.client
+server = args.server
+address = args.address
+from_seqno = args.from_seqno
+
+
+hardforks = get_config()['validator']['hardforks']
+keep_hardforks = [h for h in hardforks if h['seqno'] <= from_seqno]
+hardforks = [h for h in hardforks if h['seqno'] > from_seqno]
+if len(hardforks) == 0:
+    logger.info("No hardforks to process.")
+    sys.exit(0)
+set_hardforks(keep_hardforks)
+
+while True:
+    if len(hardforks) == 0:
+        break
+    try:
+        last_mc_seqno = get_last_mc_seqno()
+        if not last_mc_seqno:
+            logger.info("Waiting for last mc seqno...")
+            time.sleep(300)
+            continue
+        if last_mc_seqno != hardforks[0]['seqno'] - 1:
+            logger.info(f"Waiting for hardfork {hardforks[0]['seqno']}...")
+            time.sleep(300)
+            continue
+        hardfork = hardforks.pop(0)
+        logger.info(f"Processing hardfork {hardfork['seqno']}.")
+        add_hardfork(hardfork)
+        logger.info(f"Hardfork {hardfork['seqno']} has been added.")
+        restart_node()
+        logger.info(f"Node is restarted")
+    except Exception as e:
+        import traceback
+        logger.error(f"Exception occurred: {e}\n{traceback.format_exc()}")
+    time.sleep(60)
+
+logger.info(f"All done.")