add state ttl setting to cli installer

Author: yungwine

mytoninstaller/settings.py

@@ -49,6 +49,10 @@ def FirstNodeSettings(local):
 		archive_ttl = int(os.getenv('ARCHIVE_TTL'))
 	else:
 		archive_ttl = 2592000 if local.buffer.mode == 'liteserver' else 86400
+	state_ttl = None
+	if os.getenv('STATE_TTL'):
+		state_ttl = int(os.getenv('STATE_TTL'))
+		archive_ttl -= state_ttl
 
 	# Проверить конфигурацию
 	if os.path.isfile(vconfig_path):
@@ -74,12 +78,15 @@ def FirstNodeSettings(local):
 	cpus = psutil.cpu_count() - 1
 
 	ttl_cmd = ''
-	if archive_ttl == -1 and local.buffer.mode == 'liteserver':
+	if archive_ttl == -1:
 		archive_ttl = 10**9
-		ttl_cmd = f' --state-ttl {10**9} --permanent-celldb'
+		state_ttl = 10**9
+		ttl_cmd += f' --permanent-celldb'
+	if state_ttl is not None:
+		ttl_cmd += f' --state-ttl {state_ttl}'
+	ttl_cmd += f' --archive-ttl {archive_ttl}'
 
-
-	cmd = f"{validatorAppPath} --threads {cpus} --daemonize --global-config {globalConfigPath} --db {ton_db_dir} --logname {tonLogPath} --archive-ttl {archive_ttl} --verbosity 1"
+	cmd = f"{validatorAppPath} --threads {cpus} --daemonize --global-config {globalConfigPath} --db {ton_db_dir} --logname {tonLogPath} --verbosity 1"
 	cmd += ttl_cmd
 
 	if os.getenv('ADD_SHARD'):
@@ -189,12 +196,6 @@ def parse_block_value(local, block: str):
 	return int(data['seqno'])
 
 
-def process_hardforks(local):
-
-	pass
-
-
-
 def download_archive_from_ts(local):
 	archive_blocks = os.getenv('ARCHIVE_BLOCKS')
 	if archive_blocks is None:
@@ -203,7 +204,7 @@ def download_archive_from_ts(local):
 	if len(archive_blocks.split()) > 1:
 		block_from, block_to = archive_blocks.split()
 	block_from, block_to = parse_block_value(local, block_from), parse_block_value(local, block_to)
-	block_from = max(0, block_from - 100)  # to download previous package as node may require some blocks from it
+	block_from = max(1, block_from - 100)  # to download previous package as node may require some blocks from it
 
 	enable_ton_storage(local)
 	url = 'https://archival-dump.ton.org/index/mainnet.json'
@@ -244,7 +245,7 @@ def download_archive_from_ts(local):
 	update_init_block(local, state_bag['at_block'])
 	estimated_size = len(block_bags) * 4 * 2**30 + len(master_block_bags) * 4 * 2**30 * 0.2  # 4 GB per bag, 20% for master blocks
 
-	local.add_log(f"Downloading archive blocks. Rough estimate total blocks size is {estimated_size / 2**30} GB", "info")
+	local.add_log(f"Downloading archive blocks. Rough estimate total blocks size is {int(estimated_size / 2**30)} GB", "info")
 	with ThreadPoolExecutor(max_workers=4) as executor:
 		futures = [executor.submit(download_blocks, local, bag) for bag in block_bags]
 		futures += [executor.submit(download_master_blocks, local, bag) for bag in master_block_bags]
@@ -255,7 +256,7 @@ def download_archive_from_ts(local):
 				local.add_log(f"Error while downloading blocks: {e}", "error")
 				return
 
-	local.add_log(f"Downloading blocks is completed", "info")
+	local.add_log(f"Downloading blocks is completed, moving files", "info")
 
 	archive_dir = local.buffer.ton_db_dir + 'archive/'
 	import_dir = local.buffer.ton_db_dir + 'import/'

scripts/install.py

@@ -9,7 +9,7 @@
 def get_archive_ttl_message(answers: dict):
     archive_blocks = answers.get('archive-blocks')
     if not archive_blocks and answers['archive-blocks'] != 0:
-        return 'Send the number of seconds to keep the block data in the node database. Default is 2592000 (30 days)'
+        return 'Send the number of seconds to keep the blocks data in the node database. Press Enter to use default: 2592000 (30 days)'
     block_from = archive_blocks.split()[0]
     # or sent -1 to store downloaded blocks always
     if block_from.isdigit():
@@ -18,12 +18,14 @@ def get_archive_ttl_message(answers: dict):
         if answers['network'] == 'Testnet':
             url = url.replace('toncenter.com', 'testnet.toncenter.com')
         data = requests.get(url).json()
+        if not data['ok']:
+            raise Exception(f'Failed to get block: {data}')
         utime = int(data['result']['gen_utime'])
     else:
         utime = int(datetime.datetime.strptime(block_from, '%Y-%m-%d').timestamp())
     default_archive_ttl = int(time.time() - (utime - datetime.timedelta(days=30).total_seconds()))
-    answers['archive-ttl-default'] = default_archive_ttl
-    return f'Send the number of seconds to keep the block data in the node database.\nSkip to use default: {default_archive_ttl} to keep blocks from provided date for 30 days ({datetime.datetime.fromtimestamp(utime - datetime.timedelta(days=30).total_seconds())})\nOr send -1 to keep downloaded blocks always (recommended).'
+    answers['archive-ttl-default'] = '-1'
+    return f'Send the number of seconds to keep blocks in the node database. Press Enter to keep downloaded blocks always (recommended).\nFor your reference you can use TTL `{default_archive_ttl}` to keep blocks from provided block for 30 days (since {datetime.datetime.fromtimestamp(utime - datetime.timedelta(days=30).total_seconds())}).\nNote: in case you want to keep blocks garbage collection the node will sync 8-10 times slower.'
 
 
 def is_valid_date_format(date_str):
@@ -63,13 +65,22 @@ def validate_digits_or_empty(value):
         return True
     try:
         int(value)
-        if int(value) != -1 and int(value) <= 0:
+        if int(value) <= 0:
             return "Input must be a positive number"
         return True
     except ValueError:
         return "Input must be a number"
 
 
+def validate_state_ttl(value, archive_ttl):
+    v = validate_digits_or_empty(value)
+    if v is not True: return v
+    if archive_ttl and value and int(value) > int(archive_ttl):
+        return "State TTL cannot be greater than blocks TTL"
+    return True
+
+
+
 def validate_shard_format(value):
     if not value:
         return True
@@ -108,12 +119,13 @@ def run_cli():
     if mode != "validator":
         archive_blocks = questionary.text(
             "Do you want to download archive blocks via TON Storage? Press Enter to skip.\n"
-            "If yes, provide block seqno or date to start from and (optionally) block seqno or date to end with (send 0 to download all blocks and setup full archive node).\n"
+            "If yes, provide block seqno or date to start from and (optionally) block seqno or date to end with (send `1` to download all blocks and setup full archive node).\n"
             "Examples: `30850000`, `10000000 10200000`, `2025-01-01`, `2025-01-01 2025-01-30`",
             validate=validate_archive_blocks
         ).unsafe_ask()
 
     archive_ttl = None
+    state_ttl = None
     if mode == "liteserver":
         temp_answers = {
             'archive-blocks': archive_blocks,
@@ -125,6 +137,11 @@ def run_cli():
         ).unsafe_ask()
         if not archive_ttl and 'archive-ttl-default' in temp_answers:
             archive_ttl = temp_answers['archive-ttl-default']
+        if archive_ttl != '-1':
+            state_ttl = questionary.text(
+                'Send the number of seconds to keep blocks states in the node database. Press Enter to use default: 86400 (24 hours)',
+                validate= lambda x: validate_state_ttl(x, archive_ttl)
+            ).unsafe_ask()
 
     dump = None
     if not archive_blocks:
@@ -134,8 +151,8 @@ def run_cli():
         ).unsafe_ask()
 
     add_shard = questionary.text(
-        "Set shards node will sync. Skip to sync all shards. "
-        "Format: <workchain>:<shard>. Divide multiple shards with space. "
+        "Set shards node will sync. Press Enter to sync all shards.\n"
+        "Format: <workchain>:<shard>. Divide multiple shards with space.\n"
         "Example: `0:2000000000000000 0:6000000000000000`",
         validate=validate_shard_format
     ).unsafe_ask()
@@ -151,6 +168,7 @@ def run_cli():
         "validator-mode": validator_mode,
         "archive-blocks": archive_blocks,
         "archive-ttl": archive_ttl,
+        'state-ttl': state_ttl,
         "dump": dump,
         "add-shard": add_shard,
         "background": background
@@ -159,11 +177,12 @@ def run_cli():
     return answers
 
 
-def run_install(answers: dict) -> list:
+def run_install(answers: dict):
     mode = answers["mode"]
     network = answers["network"].lower()
     config = answers["config"]
     archive_ttl = answers["archive-ttl"]
+    state_ttl = answers["state-ttl"]
     add_shard = answers["add-shard"]
     validator_mode = answers["validator-mode"]
     archive_blocks = answers["archive-blocks"]
@@ -181,6 +200,8 @@ def run_install(answers: dict) -> list:
 
     if archive_ttl:
         os.environ['ARCHIVE_TTL'] = archive_ttl  # set env variable
+    if state_ttl:
+        os.environ['STATE_TTL'] = state_ttl
     if add_shard:
         os.environ['ADD_SHARD'] = add_shard
     if archive_blocks: