From 35c64c23e84ecda746568101506dd72e8e7dadee Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Fri, 12 Jul 2024 15:53:32 +0300 Subject: [PATCH 001/206] fix readme for master --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index aaf1d762..19879e13 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ ![GitHub stars](https://img.shields.io/github/stars/ton-blockchain/mytonctrl?style=flat-square&logo=github) ![GitHub forks](https://img.shields.io/github/forks/ton-blockchain/mytonctrl?style=flat-square&logo=github) ![GitHub issues](https://img.shields.io/github/issues/ton-blockchain/mytonctrl?style=flat-square&logo=github) ![GitHub pull requests](https://img.shields.io/github/issues-pr/ton-blockchain/mytonctrl?style=flat-square&logo=github) ![GitHub last commit](https://img.shields.io/github/last-commit/ton-blockchain/mytonctrl?style=flat-square&logo=github) ![GitHub license](https://img.shields.io/github/license/ton-blockchain/mytonctrl?style=flat-square&logo=github) +# Tonstakers +This fork focuses on supporting the second version of the mytonctrl utility, as well as testing installation script changes, which will later be submitted upstream. + # MyTonCtrl [Данный текст доступен на русском языке.](README.Ru.md) From 100d22098e5508ec6f6f329c42f017b5ea7895e4 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Mon, 15 Jul 2024 09:45:46 +0300 Subject: [PATCH 002/206] test hardcode work --- scripts/install.sh | 18 ++++++++++++++---- scripts/ton_installer.sh | 7 ++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index cfe84c76..34c8257c 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -12,8 +12,13 @@ if [ "$(id -u)" != "0" ]; then exit 1 fi -author="ton-blockchain" -repo="mytonctrl" +#author="ton-blockchain" +#repo="mytonctrl" +#branch="master" +#mode="validator" + +author="tonstakers" +repo="mytonctrl-v2" branch="master" mode="validator" @@ -36,7 +41,10 @@ if [[ "${1-}" =~ ^-*h(elp)?$ ]]; then fi # node install parameters -config="https://ton-blockchain.github.io/global.config.json" +# todo support parametr +#config="https://ton-blockchain.github.io/global.config.json" +config="https://ton-blockchain.github.io/testnet-global.config.json" + telemetry=true ignore=false dump=false @@ -91,7 +99,9 @@ file3=${BIN_DIR}/ton/validator-engine-console/validator-engine-console if [ ! -f "${file1}" ] || [ ! -f "${file2}" ] || [ ! -f "${file3}" ]; then echo "TON does not exists, building" - wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/ton_installer.sh -O /tmp/ton_installer.sh +# wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/ton_installer.sh -O /tmp/ton_installer.sh + wget https://raw.githubusercontent.com/tonstakers/mytonctrl-v2/master/scripts/ton_installer.sh -O /tmp/ton_installer.sh +# todo set vas scrip patch bash /tmp/ton_installer.sh -c ${config} fi diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index 500a92f3..333ac688 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -9,7 +9,9 @@ if [ "$(id -u)" != "0" ]; then fi # Get arguments -config=https://ton-blockchain.github.io/global.config.json +# todo set vars +#config=https://ton-blockchain.github.io/global.config.json +config="https://ton-blockchain.github.io/testnet-global.config.json" while getopts c: flag do case "${flag}" in @@ -104,6 +106,9 @@ echo -e "${COLOR}[3/6]${ENDC} Preparing for compilation" cd $SOURCES_DIR rm -rf $SOURCES_DIR/ton git clone --recursive https://github.com/ton-blockchain/ton.git +cd $SOURCES_DIR/ton +git checkout 5380e6f +cd ../ git config --global --add safe.directory $SOURCES_DIR/ton # Подготавливаем папки для компиляции From 03fe23f0216d8354e8b47a8c508dd0bf4cc70d67 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Mon, 15 Jul 2024 10:17:28 +0300 Subject: [PATCH 003/206] test support vars --- scripts/install.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 34c8257c..608faf03 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -31,7 +31,8 @@ show_help_and_exit() { echo ' -a Set MyTonCtrl git repo author' echo ' -r Set MyTonCtrl git repo' echo ' -b Set MyTonCtrl git repo branch' - echo ' -m MODE Install MyTonCtrl with specified mode (validator or liteserver)' + echo ' -m MODE Install MyTonCtrl with specified mode (validator or liteserver)' + echo ' -n NETWORK Specify the network (mainnet or testnet)' echo ' -h Show this help' exit } @@ -41,15 +42,12 @@ if [[ "${1-}" =~ ^-*h(elp)?$ ]]; then fi # node install parameters -# todo support parametr -#config="https://ton-blockchain.github.io/global.config.json" -config="https://ton-blockchain.github.io/testnet-global.config.json" - +config="https://ton-blockchain.github.io/global.config.json" telemetry=true ignore=false dump=false - - +cpu_required=16 +mem_required=64000000 # 64GB in KB while getopts c:tida:r:b:m: flag do @@ -62,6 +60,7 @@ do r) repo=${OPTARG};; b) branch=${OPTARG};; m) mode=${OPTARG};; + n) network=${OPTARG};; h) show_help_and_exit;; *) echo "Flag -${flag} is not recognized. Aborting" @@ -69,6 +68,13 @@ do esac done +# Set config based on network argument +if [ "${network}" = "testnet" ]; then + config="https://ton-blockchain.github.io/testnet-global.config.json" + cpu_required=8 + mem_required=16000000 # 16GB in KB +fi + # check machine configuration echo -e "${COLOR}[1/5]${ENDC} Checking system requirements" @@ -76,8 +82,8 @@ cpus=$(lscpu | grep "CPU(s)" | head -n 1 | awk '{print $2}') memory=$(cat /proc/meminfo | grep MemTotal | awk '{print $2}') echo "This machine has ${cpus} CPUs and ${memory}KB of Memory" -if [ "$ignore" = false ] && ([ "${cpus}" -lt 16 ] || [ "${memory}" -lt 64000000 ]); then - echo "Insufficient resources. Requires a minimum of 16 processors and 64Gb RAM." +if [ "$ignore" = false ] && ([ "${cpus}" -lt "${cpu_required}" ] || [ "${memory}" -lt "${mem_required}"]); then + echo "Insufficient resources. Requires a minimum of "${cpu_required}" processors and "${mem_required}" RAM." exit 1 fi From 13ea62383a8515f89cfbd6d178fd4c138a2afa71 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Mon, 15 Jul 2024 10:22:52 +0300 Subject: [PATCH 004/206] set ton install default work --- scripts/install.sh | 4 +--- scripts/ton_installer.sh | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 608faf03..d89e3317 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -105,9 +105,7 @@ file3=${BIN_DIR}/ton/validator-engine-console/validator-engine-console if [ ! -f "${file1}" ] || [ ! -f "${file2}" ] || [ ! -f "${file3}" ]; then echo "TON does not exists, building" -# wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/ton_installer.sh -O /tmp/ton_installer.sh - wget https://raw.githubusercontent.com/tonstakers/mytonctrl-v2/master/scripts/ton_installer.sh -O /tmp/ton_installer.sh -# todo set vas scrip patch + wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/ton_installer.sh -O /tmp/ton_installer.sh bash /tmp/ton_installer.sh -c ${config} fi diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index 333ac688..dfa5edde 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -2,6 +2,10 @@ #!/bin/bash set -e +#todo remove - debug only +# fork tonstackers + + # Проверить sudo if [ "$(id -u)" != "0" ]; then echo "Please run script as root" From e599003ce755b227e0d8f239d9bb0f30711c0fa0 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Mon, 15 Jul 2024 10:29:02 +0300 Subject: [PATCH 005/206] scriup update --- scripts/install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index d89e3317..411f34b9 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -32,7 +32,7 @@ show_help_and_exit() { echo ' -r Set MyTonCtrl git repo' echo ' -b Set MyTonCtrl git repo branch' echo ' -m MODE Install MyTonCtrl with specified mode (validator or liteserver)' - echo ' -n NETWORK Specify the network (mainnet or testnet)' + echo ' -n NETWORK Specify the network (mainnet or testnet)' echo ' -h Show this help' exit } @@ -59,8 +59,8 @@ do a) author=${OPTARG};; r) repo=${OPTARG};; b) branch=${OPTARG};; - m) mode=${OPTARG};; - n) network=${OPTARG};; + m) mode=${OPTARG};; + n) network=${OPTARG};; h) show_help_and_exit;; *) echo "Flag -${flag} is not recognized. Aborting" From 1297fe26c72a36d92985cc4cb28a0bc2860c6ab5 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Mon, 15 Jul 2024 10:32:20 +0300 Subject: [PATCH 006/206] scriup update --- scripts/install.sh | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 411f34b9..90518e70 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -21,20 +21,21 @@ author="tonstakers" repo="mytonctrl-v2" branch="master" mode="validator" +network="mainnet" show_help_and_exit() { - echo 'Supported argumets:' + echo 'Supported arguments:' echo ' -c PATH Provide custom config for toninstaller.sh' echo ' -t Disable telemetry' - echo ' -i Ignore minimum reqiurements' + echo ' -i Ignore minimum requirements' echo ' -d Use pre-packaged dump. Reduces duration of initial synchronization.' echo ' -a Set MyTonCtrl git repo author' - echo ' -r Set MyTonCtrl git repo' - echo ' -b Set MyTonCtrl git repo branch' - echo ' -m MODE Install MyTonCtrl with specified mode (validator or liteserver)' + echo ' -r Set MyTonCtrl git repo' + echo ' -b Set MyTonCtrl git repo branch' + echo ' -m MODE Install MyTonCtrl with specified mode (validator or liteserver)' echo ' -n NETWORK Specify the network (mainnet or testnet)' - echo ' -h Show this help' - exit + echo ' -h Show this help' + exit } if [[ "${1-}" =~ ^-*h(elp)?$ ]]; then @@ -49,23 +50,22 @@ dump=false cpu_required=16 mem_required=64000000 # 64GB in KB -while getopts c:tida:r:b:m: flag -do - case "${flag}" in - c) config=${OPTARG};; - t) telemetry=false;; - i) ignore=true;; - d) dump=true;; - a) author=${OPTARG};; - r) repo=${OPTARG};; - b) branch=${OPTARG};; - m) mode=${OPTARG};; - n) network=${OPTARG};; - h) show_help_and_exit;; - *) +while getopts ":c:tida:r:b:m:n:h" flag; do + case "${flag}" in + c) config=${OPTARG};; + t) telemetry=false;; + i) ignore=true;; + d) dump=true;; + a) author=${OPTARG};; + r) repo=${OPTARG};; + b) branch=${OPTARG};; + m) mode=${OPTARG};; + n) network=${OPTARG};; + h) show_help_and_exit;; + *) echo "Flag -${flag} is not recognized. Aborting" exit 1 ;; - esac + esac done # Set config based on network argument From 6b082a9daa01a8d975481962662552452f56d4b1 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Mon, 15 Jul 2024 10:59:02 +0300 Subject: [PATCH 007/206] scriup update --- scripts/install.sh | 5 +++++ scripts/ton_installer.sh | 3 --- scripts/uninstall.sh | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 90518e70..3814dff5 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -143,5 +143,10 @@ mkdir -p ${version_dir} echo ${migrate_version} > ${version_path} chown ${user}:${user} ${version_dir} ${version_path} +# create symbolic link if branch not eq mytonctrl +if [ "${repo}" != "mytonctrl" ]; then + ln -sf ${SOURCES_DIR}/${repo} ${SOURCES_DIR}/mytonctrl +fi + echo -e "${COLOR}[5/5]${ENDC} Mytonctrl installation completed" exit 0 diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index dfa5edde..201bc709 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -13,9 +13,6 @@ if [ "$(id -u)" != "0" ]; then fi # Get arguments -# todo set vars -#config=https://ton-blockchain.github.io/global.config.json -config="https://ton-blockchain.github.io/testnet-global.config.json" while getopts c: flag do case "${flag}" in diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index 9e1f275c..babfd176 100644 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -41,6 +41,7 @@ if $full; then fi rm -rf /usr/src/mytonctrl +rm -rf /usr/src/mytonctrl-v2 #todo remove rm -rf /usr/src/mtc-jsonrpc rm -rf /usr/src/pytonv3 rm -rf /tmp/myton* From 9f673a7a79d02613704c8dadecfcef4c79560c18 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Mon, 15 Jul 2024 11:08:24 +0300 Subject: [PATCH 008/206] scriup update --- scripts/install.sh | 8 ++++++-- scripts/ton_installer.sh | 34 ++++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 3814dff5..10bd354c 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -22,6 +22,8 @@ repo="mytonctrl-v2" branch="master" mode="validator" network="mainnet" +ton_node_version="master" # Default version + show_help_and_exit() { echo 'Supported arguments:' @@ -34,6 +36,7 @@ show_help_and_exit() { echo ' -b Set MyTonCtrl git repo branch' echo ' -m MODE Install MyTonCtrl with specified mode (validator or liteserver)' echo ' -n NETWORK Specify the network (mainnet or testnet)' + echo ' -v VERSION Specify the ton node version (commit, branch, or tag)' echo ' -h Show this help' exit } @@ -50,7 +53,7 @@ dump=false cpu_required=16 mem_required=64000000 # 64GB in KB -while getopts ":c:tida:r:b:m:n:h" flag; do +while getopts ":c:tida:r:b:m:n:v:h" flag; do case "${flag}" in c) config=${OPTARG};; t) telemetry=false;; @@ -61,6 +64,7 @@ while getopts ":c:tida:r:b:m:n:h" flag; do b) branch=${OPTARG};; m) mode=${OPTARG};; n) network=${OPTARG};; + v) ton_node_version=${OPTARG};; h) show_help_and_exit;; *) echo "Flag -${flag} is not recognized. Aborting" @@ -106,7 +110,7 @@ file3=${BIN_DIR}/ton/validator-engine-console/validator-engine-console if [ ! -f "${file1}" ] || [ ! -f "${file2}" ] || [ ! -f "${file3}" ]; then echo "TON does not exists, building" wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/ton_installer.sh -O /tmp/ton_installer.sh - bash /tmp/ton_installer.sh -c ${config} + bash /tmp/ton_installer.sh -c ${config} -v ${ton_node_version} fi # Cloning mytonctrl diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index 201bc709..d0ec15cc 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -13,13 +13,25 @@ if [ "$(id -u)" != "0" ]; then fi # Get arguments -while getopts c: flag -do - case "${flag}" in - c) config=${OPTARG};; - esac +#while getopts c: flag +#do +# case "${flag}" in +# c) config=${OPTARG};; +# esac +#done + +while getopts ":c:v" flag; do + case "${flag}" in + c) config=${OPTARG};; + v) ton_node_version=${OPTARG};; + h) show_help_and_exit;; + *) + echo "Flag -${flag} is not recognized. Aborting" + exit 1 ;; + esac done + # Цвета COLOR='\033[95m' ENDC='\033[0m' @@ -107,9 +119,15 @@ echo -e "${COLOR}[3/6]${ENDC} Preparing for compilation" cd $SOURCES_DIR rm -rf $SOURCES_DIR/ton git clone --recursive https://github.com/ton-blockchain/ton.git -cd $SOURCES_DIR/ton -git checkout 5380e6f -cd ../ + +echo "checkout to ${ton_node_version}" + +if [ "${ton_node_version}" != "master" ]; then + cd $SOURCES_DIR/ton + git checkout ${ton_node_version} + cd ../ +fi + git config --global --add safe.directory $SOURCES_DIR/ton # Подготавливаем папки для компиляции From 2a159fc50e41865bf6e2f3a2ed15acb21afd6563 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Mon, 15 Jul 2024 11:30:54 +0300 Subject: [PATCH 009/206] scriup update --- scripts/ton_installer.sh | 305 ++++++++++++++++++++------------------- 1 file changed, 153 insertions(+), 152 deletions(-) diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index d0ec15cc..06f297e5 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -31,156 +31,157 @@ while getopts ":c:v" flag; do esac done - -# Цвета -COLOR='\033[95m' -ENDC='\033[0m' - -# На OSX нет такой директории по-умолчанию, поэтому создаем... -SOURCES_DIR=/usr/src -BIN_DIR=/usr/bin -if [[ "$OSTYPE" =~ darwin.* ]]; then - SOURCES_DIR=/usr/local/src - BIN_DIR=/usr/local/bin - mkdir -p $SOURCES_DIR -fi - -# Установка требуемых пакетов -echo -e "${COLOR}[1/6]${ENDC} Installing required packages" -if [ "$OSTYPE" == "linux-gnu" ]; then - if [ hash yum 2>/dev/null ]; then - echo "RHEL-based Linux detected." - yum install -y epel-release - dnf config-manager --set-enabled PowerTools - yum install -y curl git make cmake clang gflags gflags-devel zlib zlib-devel openssl-devel openssl-libs readline-devel libmicrohttpd python3 python3-pip python36-devel - elif [ -f /etc/SuSE-release ]; then - echo "Suse Linux detected." - echo "This OS is not supported with this script at present. Sorry." - echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." - exit 1 - elif [ -f /etc/arch-release ]; then - echo "Arch Linux detected." - pacman -Syuy - pacman -S --noconfirm curl git make cmake clang gflags zlib openssl readline libmicrohttpd python python-pip - elif [ -f /etc/debian_version ]; then - echo "Ubuntu/Debian Linux detected." - apt-get update - apt-get install -y build-essential curl git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev - - # Install ninja - apt-get install -y ninja-build - - # Install for benchmark - apt install -y fio rocksdb-tools - else - echo "Unknown Linux distribution." - echo "This OS is not supported with this script at present. Sorry." - echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." - exit 1 - fi -elif [[ "$OSTYPE" =~ darwin.* ]]; then - echo "Mac OS (Darwin) detected." - if [ ! which brew >/dev/null 2>&1 ]; then - $BIN_DIR/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - fi - - echo "Please, write down your username, because brew package manager cannot be run under root user:" - read LOCAL_USERNAME - - su $LOCAL_USERNAME -c "brew update" - su $LOCAL_USERNAME -c "brew install openssl cmake llvm" -elif [ "$OSTYPE" == "freebsd"* ]; then - echo "FreeBSD detected." - echo "This OS is not supported with this script at present. Sorry." - echo "Please refer to https://github.com/paritytech/substrate for setup information." # TODO: remove links - exit 1 -else - echo "Unknown operating system." - echo "This OS is not supported with this script at present. Sorry." - echo "Please refer to https://github.com/paritytech/substrate for setup information." # TODO: remove links - exit 1 -fi - -# Установка компонентов python3 -pip3 install psutil crc16 requests - -# build openssl 3.0 -echo -e "${COLOR}[2/6]${ENDC} Building OpenSSL 3.0" -rm -rf $BIN_DIR/openssl_3 -git clone https://github.com/openssl/openssl $BIN_DIR/openssl_3 -cd $BIN_DIR/openssl_3 -opensslPath=`pwd` -git checkout openssl-3.1.4 -./config -make build_libs -j$(nproc) - -# Клонирование репозиториев с github.com -echo -e "${COLOR}[3/6]${ENDC} Preparing for compilation" -cd $SOURCES_DIR -rm -rf $SOURCES_DIR/ton -git clone --recursive https://github.com/ton-blockchain/ton.git - echo "checkout to ${ton_node_version}" - -if [ "${ton_node_version}" != "master" ]; then - cd $SOURCES_DIR/ton - git checkout ${ton_node_version} - cd ../ -fi - -git config --global --add safe.directory $SOURCES_DIR/ton - -# Подготавливаем папки для компиляции -rm -rf $BIN_DIR/ton -mkdir $BIN_DIR/ton -cd $BIN_DIR/ton - -# Подготовиться к компиляции -if [[ "$OSTYPE" =~ darwin.* ]]; then - export CMAKE_C_COMPILER=$(which clang) - export CMAKE_CXX_COMPILER=$(which clang++) - export CCACHE_DISABLE=1 -else - export CC=$(which clang) - export CXX=$(which clang++) - export CCACHE_DISABLE=1 -fi - -# Подготовиться к компиляции -if [[ "$OSTYPE" =~ darwin.* ]]; then - if [[ $(uname -p) == 'arm' ]]; then - echo M1 - CC="clang -mcpu=apple-a14" CXX="clang++ -mcpu=apple-a14" cmake $SOURCES_DIR/ton -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= -Wno-dev - else - cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton - fi -else - cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja -DTON_USE_JEMALLOC=ON -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a -fi - -# Расчитываем количество процессоров для сборки -if [[ "$OSTYPE" =~ darwin.* ]]; then - cpu_number=$(sysctl -n hw.logicalcpu) -else - memory=$(cat /proc/meminfo | grep MemAvailable | awk '{print $2}') - cpu_number=$(($memory/2100000)) - max_cpu_number=$(nproc) - if [ ${cpu_number} -gt ${max_cpu_number} ]; then - cpu_number=$((${max_cpu_number}-1)) - fi - if [ ${cpu_number} == 0 ]; then - echo "Warning! insufficient RAM" - cpu_number=1 - fi -fi - -echo -e "${COLOR}[4/6]${ENDC} Source compilation, use ${cpu_number} cpus" -ninja -j ${cpu_number} fift validator-engine lite-client validator-engine-console generate-random-id dht-server func tonlibjson rldp-http-proxy - -# Скачиваем конфигурационные файлы lite-client -echo -e "${COLOR}[5/6]${ENDC} Downloading config files" -wget ${config} -O global.config.json - -# Выход из программы -echo -e "${COLOR}[6/6]${ENDC} TON software installation complete" -exit 0 +# +## Цвета +#COLOR='\033[95m' +#ENDC='\033[0m' +# +## На OSX нет такой директории по-умолчанию, поэтому создаем... +#SOURCES_DIR=/usr/src +#BIN_DIR=/usr/bin +#if [[ "$OSTYPE" =~ darwin.* ]]; then +# SOURCES_DIR=/usr/local/src +# BIN_DIR=/usr/local/bin +# mkdir -p $SOURCES_DIR +#fi +# +## Установка требуемых пакетов +#echo -e "${COLOR}[1/6]${ENDC} Installing required packages" +#if [ "$OSTYPE" == "linux-gnu" ]; then +# if [ hash yum 2>/dev/null ]; then +# echo "RHEL-based Linux detected." +# yum install -y epel-release +# dnf config-manager --set-enabled PowerTools +# yum install -y curl git make cmake clang gflags gflags-devel zlib zlib-devel openssl-devel openssl-libs readline-devel libmicrohttpd python3 python3-pip python36-devel +# elif [ -f /etc/SuSE-release ]; then +# echo "Suse Linux detected." +# echo "This OS is not supported with this script at present. Sorry." +# echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." +# exit 1 +# elif [ -f /etc/arch-release ]; then +# echo "Arch Linux detected." +# pacman -Syuy +# pacman -S --noconfirm curl git make cmake clang gflags zlib openssl readline libmicrohttpd python python-pip +# elif [ -f /etc/debian_version ]; then +# echo "Ubuntu/Debian Linux detected." +# apt-get update +# apt-get install -y build-essential curl git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev +# +# # Install ninja +# apt-get install -y ninja-build +# +# # Install for benchmark +# apt install -y fio rocksdb-tools +# else +# echo "Unknown Linux distribution." +# echo "This OS is not supported with this script at present. Sorry." +# echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." +# exit 1 +# fi +#elif [[ "$OSTYPE" =~ darwin.* ]]; then +# echo "Mac OS (Darwin) detected." +# if [ ! which brew >/dev/null 2>&1 ]; then +# $BIN_DIR/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +# fi +# +# echo "Please, write down your username, because brew package manager cannot be run under root user:" +# read LOCAL_USERNAME +# +# su $LOCAL_USERNAME -c "brew update" +# su $LOCAL_USERNAME -c "brew install openssl cmake llvm" +#elif [ "$OSTYPE" == "freebsd"* ]; then +# echo "FreeBSD detected." +# echo "This OS is not supported with this script at present. Sorry." +# echo "Please refer to https://github.com/paritytech/substrate for setup information." # TODO: remove links +# exit 1 +#else +# echo "Unknown operating system." +# echo "This OS is not supported with this script at present. Sorry." +# echo "Please refer to https://github.com/paritytech/substrate for setup information." # TODO: remove links +# exit 1 +#fi +# +## Установка компонентов python3 +#pip3 install psutil crc16 requests +# +## build openssl 3.0 +#echo -e "${COLOR}[2/6]${ENDC} Building OpenSSL 3.0" +#rm -rf $BIN_DIR/openssl_3 +#git clone https://github.com/openssl/openssl $BIN_DIR/openssl_3 +#cd $BIN_DIR/openssl_3 +#opensslPath=`pwd` +#git checkout openssl-3.1.4 +#./config +#make build_libs -j$(nproc) +# +## Клонирование репозиториев с github.com +#echo -e "${COLOR}[3/6]${ENDC} Preparing for compilation" +#cd $SOURCES_DIR +#rm -rf $SOURCES_DIR/ton +#git clone --recursive https://github.com/ton-blockchain/ton.git +# +#echo "checkout to ${ton_node_version}" +# +#if [ "${ton_node_version}" != "master" ]; then +# cd $SOURCES_DIR/ton +# git checkout ${ton_node_version} +# cd ../ +#fi +# +#git config --global --add safe.directory $SOURCES_DIR/ton +# +## Подготавливаем папки для компиляции +#rm -rf $BIN_DIR/ton +#mkdir $BIN_DIR/ton +#cd $BIN_DIR/ton +# +## Подготовиться к компиляции +#if [[ "$OSTYPE" =~ darwin.* ]]; then +# export CMAKE_C_COMPILER=$(which clang) +# export CMAKE_CXX_COMPILER=$(which clang++) +# export CCACHE_DISABLE=1 +#else +# export CC=$(which clang) +# export CXX=$(which clang++) +# export CCACHE_DISABLE=1 +#fi +# +## Подготовиться к компиляции +#if [[ "$OSTYPE" =~ darwin.* ]]; then +# if [[ $(uname -p) == 'arm' ]]; then +# echo M1 +# CC="clang -mcpu=apple-a14" CXX="clang++ -mcpu=apple-a14" cmake $SOURCES_DIR/ton -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= -Wno-dev +# else +# cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton +# fi +#else +# cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja -DTON_USE_JEMALLOC=ON -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a +#fi +# +## Расчитываем количество процессоров для сборки +#if [[ "$OSTYPE" =~ darwin.* ]]; then +# cpu_number=$(sysctl -n hw.logicalcpu) +#else +# memory=$(cat /proc/meminfo | grep MemAvailable | awk '{print $2}') +# cpu_number=$(($memory/2100000)) +# max_cpu_number=$(nproc) +# if [ ${cpu_number} -gt ${max_cpu_number} ]; then +# cpu_number=$((${max_cpu_number}-1)) +# fi +# if [ ${cpu_number} == 0 ]; then +# echo "Warning! insufficient RAM" +# cpu_number=1 +# fi +#fi +# +#echo -e "${COLOR}[4/6]${ENDC} Source compilation, use ${cpu_number} cpus" +#ninja -j ${cpu_number} fift validator-engine lite-client validator-engine-console generate-random-id dht-server func tonlibjson rldp-http-proxy +# +## Скачиваем конфигурационные файлы lite-client +#echo -e "${COLOR}[5/6]${ENDC} Downloading config files" +#wget ${config} -O global.config.json +# +## Выход из программы +#echo -e "${COLOR}[6/6]${ENDC} TON software installation complete" +#exit 0 From 524db00751982f6fb209626996be64746e1e4c54 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Mon, 15 Jul 2024 11:32:49 +0300 Subject: [PATCH 010/206] scriup update --- scripts/ton_installer.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index 06f297e5..5456ffc2 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -31,6 +31,7 @@ while getopts ":c:v" flag; do esac done +echo " config 42: ${config}" echo "checkout to ${ton_node_version}" # ## Цвета From 80dc5a2e9f893ecdf9232cad1937aa6ce0b45cf2 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Mon, 15 Jul 2024 11:36:50 +0300 Subject: [PATCH 011/206] scriup update --- scripts/ton_installer.sh | 316 +++++++++++++++++++-------------------- 1 file changed, 154 insertions(+), 162 deletions(-) diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index 5456ffc2..680d1835 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -12,15 +12,7 @@ if [ "$(id -u)" != "0" ]; then exit 1 fi -# Get arguments -#while getopts c: flag -#do -# case "${flag}" in -# c) config=${OPTARG};; -# esac -#done - -while getopts ":c:v" flag; do +while getopts ":c:v:h" flag; do case "${flag}" in c) config=${OPTARG};; v) ton_node_version=${OPTARG};; @@ -33,156 +25,156 @@ done echo " config 42: ${config}" echo "checkout to ${ton_node_version}" -# -## Цвета -#COLOR='\033[95m' -#ENDC='\033[0m' -# -## На OSX нет такой директории по-умолчанию, поэтому создаем... -#SOURCES_DIR=/usr/src -#BIN_DIR=/usr/bin -#if [[ "$OSTYPE" =~ darwin.* ]]; then -# SOURCES_DIR=/usr/local/src -# BIN_DIR=/usr/local/bin -# mkdir -p $SOURCES_DIR -#fi -# -## Установка требуемых пакетов -#echo -e "${COLOR}[1/6]${ENDC} Installing required packages" -#if [ "$OSTYPE" == "linux-gnu" ]; then -# if [ hash yum 2>/dev/null ]; then -# echo "RHEL-based Linux detected." -# yum install -y epel-release -# dnf config-manager --set-enabled PowerTools -# yum install -y curl git make cmake clang gflags gflags-devel zlib zlib-devel openssl-devel openssl-libs readline-devel libmicrohttpd python3 python3-pip python36-devel -# elif [ -f /etc/SuSE-release ]; then -# echo "Suse Linux detected." -# echo "This OS is not supported with this script at present. Sorry." -# echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." -# exit 1 -# elif [ -f /etc/arch-release ]; then -# echo "Arch Linux detected." -# pacman -Syuy -# pacman -S --noconfirm curl git make cmake clang gflags zlib openssl readline libmicrohttpd python python-pip -# elif [ -f /etc/debian_version ]; then -# echo "Ubuntu/Debian Linux detected." -# apt-get update -# apt-get install -y build-essential curl git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev -# -# # Install ninja -# apt-get install -y ninja-build -# -# # Install for benchmark -# apt install -y fio rocksdb-tools -# else -# echo "Unknown Linux distribution." -# echo "This OS is not supported with this script at present. Sorry." -# echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." -# exit 1 -# fi -#elif [[ "$OSTYPE" =~ darwin.* ]]; then -# echo "Mac OS (Darwin) detected." -# if [ ! which brew >/dev/null 2>&1 ]; then -# $BIN_DIR/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -# fi -# -# echo "Please, write down your username, because brew package manager cannot be run under root user:" -# read LOCAL_USERNAME -# -# su $LOCAL_USERNAME -c "brew update" -# su $LOCAL_USERNAME -c "brew install openssl cmake llvm" -#elif [ "$OSTYPE" == "freebsd"* ]; then -# echo "FreeBSD detected." -# echo "This OS is not supported with this script at present. Sorry." -# echo "Please refer to https://github.com/paritytech/substrate for setup information." # TODO: remove links -# exit 1 -#else -# echo "Unknown operating system." -# echo "This OS is not supported with this script at present. Sorry." -# echo "Please refer to https://github.com/paritytech/substrate for setup information." # TODO: remove links -# exit 1 -#fi -# -## Установка компонентов python3 -#pip3 install psutil crc16 requests -# -## build openssl 3.0 -#echo -e "${COLOR}[2/6]${ENDC} Building OpenSSL 3.0" -#rm -rf $BIN_DIR/openssl_3 -#git clone https://github.com/openssl/openssl $BIN_DIR/openssl_3 -#cd $BIN_DIR/openssl_3 -#opensslPath=`pwd` -#git checkout openssl-3.1.4 -#./config -#make build_libs -j$(nproc) -# -## Клонирование репозиториев с github.com -#echo -e "${COLOR}[3/6]${ENDC} Preparing for compilation" -#cd $SOURCES_DIR -#rm -rf $SOURCES_DIR/ton -#git clone --recursive https://github.com/ton-blockchain/ton.git -# -#echo "checkout to ${ton_node_version}" -# -#if [ "${ton_node_version}" != "master" ]; then -# cd $SOURCES_DIR/ton -# git checkout ${ton_node_version} -# cd ../ -#fi -# -#git config --global --add safe.directory $SOURCES_DIR/ton -# -## Подготавливаем папки для компиляции -#rm -rf $BIN_DIR/ton -#mkdir $BIN_DIR/ton -#cd $BIN_DIR/ton -# -## Подготовиться к компиляции -#if [[ "$OSTYPE" =~ darwin.* ]]; then -# export CMAKE_C_COMPILER=$(which clang) -# export CMAKE_CXX_COMPILER=$(which clang++) -# export CCACHE_DISABLE=1 -#else -# export CC=$(which clang) -# export CXX=$(which clang++) -# export CCACHE_DISABLE=1 -#fi -# -## Подготовиться к компиляции -#if [[ "$OSTYPE" =~ darwin.* ]]; then -# if [[ $(uname -p) == 'arm' ]]; then -# echo M1 -# CC="clang -mcpu=apple-a14" CXX="clang++ -mcpu=apple-a14" cmake $SOURCES_DIR/ton -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= -Wno-dev -# else -# cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -# fi -#else -# cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja -DTON_USE_JEMALLOC=ON -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a -#fi -# -## Расчитываем количество процессоров для сборки -#if [[ "$OSTYPE" =~ darwin.* ]]; then -# cpu_number=$(sysctl -n hw.logicalcpu) -#else -# memory=$(cat /proc/meminfo | grep MemAvailable | awk '{print $2}') -# cpu_number=$(($memory/2100000)) -# max_cpu_number=$(nproc) -# if [ ${cpu_number} -gt ${max_cpu_number} ]; then -# cpu_number=$((${max_cpu_number}-1)) -# fi -# if [ ${cpu_number} == 0 ]; then -# echo "Warning! insufficient RAM" -# cpu_number=1 -# fi -#fi -# -#echo -e "${COLOR}[4/6]${ENDC} Source compilation, use ${cpu_number} cpus" -#ninja -j ${cpu_number} fift validator-engine lite-client validator-engine-console generate-random-id dht-server func tonlibjson rldp-http-proxy -# -## Скачиваем конфигурационные файлы lite-client -#echo -e "${COLOR}[5/6]${ENDC} Downloading config files" -#wget ${config} -O global.config.json -# -## Выход из программы -#echo -e "${COLOR}[6/6]${ENDC} TON software installation complete" -#exit 0 + +# Цвета +COLOR='\033[95m' +ENDC='\033[0m' + +# На OSX нет такой директории по-умолчанию, поэтому создаем... +SOURCES_DIR=/usr/src +BIN_DIR=/usr/bin +if [[ "$OSTYPE" =~ darwin.* ]]; then + SOURCES_DIR=/usr/local/src + BIN_DIR=/usr/local/bin + mkdir -p $SOURCES_DIR +fi + +# Установка требуемых пакетов +echo -e "${COLOR}[1/6]${ENDC} Installing required packages" +if [ "$OSTYPE" == "linux-gnu" ]; then + if [ hash yum 2>/dev/null ]; then + echo "RHEL-based Linux detected." + yum install -y epel-release + dnf config-manager --set-enabled PowerTools + yum install -y curl git make cmake clang gflags gflags-devel zlib zlib-devel openssl-devel openssl-libs readline-devel libmicrohttpd python3 python3-pip python36-devel + elif [ -f /etc/SuSE-release ]; then + echo "Suse Linux detected." + echo "This OS is not supported with this script at present. Sorry." + echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." + exit 1 + elif [ -f /etc/arch-release ]; then + echo "Arch Linux detected." + pacman -Syuy + pacman -S --noconfirm curl git make cmake clang gflags zlib openssl readline libmicrohttpd python python-pip + elif [ -f /etc/debian_version ]; then + echo "Ubuntu/Debian Linux detected." + apt-get update + apt-get install -y build-essential curl git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev + + # Install ninja + apt-get install -y ninja-build + + # Install for benchmark + apt install -y fio rocksdb-tools + else + echo "Unknown Linux distribution." + echo "This OS is not supported with this script at present. Sorry." + echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." + exit 1 + fi +elif [[ "$OSTYPE" =~ darwin.* ]]; then + echo "Mac OS (Darwin) detected." + if [ ! which brew >/dev/null 2>&1 ]; then + $BIN_DIR/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + fi + + echo "Please, write down your username, because brew package manager cannot be run under root user:" + read LOCAL_USERNAME + + su $LOCAL_USERNAME -c "brew update" + su $LOCAL_USERNAME -c "brew install openssl cmake llvm" +elif [ "$OSTYPE" == "freebsd"* ]; then + echo "FreeBSD detected." + echo "This OS is not supported with this script at present. Sorry." + echo "Please refer to https://github.com/paritytech/substrate for setup information." # TODO: remove links + exit 1 +else + echo "Unknown operating system." + echo "This OS is not supported with this script at present. Sorry." + echo "Please refer to https://github.com/paritytech/substrate for setup information." # TODO: remove links + exit 1 +fi + +# Установка компонентов python3 +pip3 install psutil crc16 requests + +# build openssl 3.0 +echo -e "${COLOR}[2/6]${ENDC} Building OpenSSL 3.0" +rm -rf $BIN_DIR/openssl_3 +git clone https://github.com/openssl/openssl $BIN_DIR/openssl_3 +cd $BIN_DIR/openssl_3 +opensslPath=`pwd` +git checkout openssl-3.1.4 +./config +make build_libs -j$(nproc) + +# Клонирование репозиториев с github.com +echo -e "${COLOR}[3/6]${ENDC} Preparing for compilation" +cd $SOURCES_DIR +rm -rf $SOURCES_DIR/ton +git clone --recursive https://github.com/ton-blockchain/ton.git + +echo "checkout to ${ton_node_version}" + +if [ "${ton_node_version}" != "master" ]; then + cd $SOURCES_DIR/ton + git checkout ${ton_node_version} + cd ../ +fi + +git config --global --add safe.directory $SOURCES_DIR/ton + +# Подготавливаем папки для компиляции +rm -rf $BIN_DIR/ton +mkdir $BIN_DIR/ton +cd $BIN_DIR/ton + +# Подготовиться к компиляции +if [[ "$OSTYPE" =~ darwin.* ]]; then + export CMAKE_C_COMPILER=$(which clang) + export CMAKE_CXX_COMPILER=$(which clang++) + export CCACHE_DISABLE=1 +else + export CC=$(which clang) + export CXX=$(which clang++) + export CCACHE_DISABLE=1 +fi + +# Подготовиться к компиляции +if [[ "$OSTYPE" =~ darwin.* ]]; then + if [[ $(uname -p) == 'arm' ]]; then + echo M1 + CC="clang -mcpu=apple-a14" CXX="clang++ -mcpu=apple-a14" cmake $SOURCES_DIR/ton -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= -Wno-dev + else + cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton + fi +else + cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja -DTON_USE_JEMALLOC=ON -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$opensslPath/include -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a +fi + +# Расчитываем количество процессоров для сборки +if [[ "$OSTYPE" =~ darwin.* ]]; then + cpu_number=$(sysctl -n hw.logicalcpu) +else + memory=$(cat /proc/meminfo | grep MemAvailable | awk '{print $2}') + cpu_number=$(($memory/2100000)) + max_cpu_number=$(nproc) + if [ ${cpu_number} -gt ${max_cpu_number} ]; then + cpu_number=$((${max_cpu_number}-1)) + fi + if [ ${cpu_number} == 0 ]; then + echo "Warning! insufficient RAM" + cpu_number=1 + fi +fi + +echo -e "${COLOR}[4/6]${ENDC} Source compilation, use ${cpu_number} cpus" +ninja -j ${cpu_number} fift validator-engine lite-client validator-engine-console generate-random-id dht-server func tonlibjson rldp-http-proxy + +# Скачиваем конфигурационные файлы lite-client +echo -e "${COLOR}[5/6]${ENDC} Downloading config files" +wget ${config} -O global.config.json + +# Выход из программы +echo -e "${COLOR}[6/6]${ENDC} TON software installation complete" +exit 0 From cea8cb492673d6bbd4ed49a1a5a4d25086e88598 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Tue, 16 Jul 2024 14:51:50 +0300 Subject: [PATCH 012/206] ## Changes - **Network support added**: Configuration is based on the network. - **TON Node version**: Added support for the TON Node version. - **Support for non-default repositories in mytonctl**: Ensures proper operation of mytonctl when executing commands with non-default repositories. - **Resource validation**: Added support for different values for testnet and mainnet. --- scripts/install.sh | 9 ++------- scripts/ton_installer.sh | 6 +----- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 10bd354c..5975351b 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -12,13 +12,8 @@ if [ "$(id -u)" != "0" ]; then exit 1 fi -#author="ton-blockchain" -#repo="mytonctrl" -#branch="master" -#mode="validator" - -author="tonstakers" -repo="mytonctrl-v2" +author="ton-blockchain" +repo="mytonctrl" branch="master" mode="validator" network="mainnet" diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index 680d1835..3ae64874 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -2,10 +2,6 @@ #!/bin/bash set -e -#todo remove - debug only -# fork tonstackers - - # Проверить sudo if [ "$(id -u)" != "0" ]; then echo "Please run script as root" @@ -23,7 +19,7 @@ while getopts ":c:v:h" flag; do esac done -echo " config 42: ${config}" +echo "config: ${config}" echo "checkout to ${ton_node_version}" # Цвета From e443f8268ed729bf63b5cc2d37011fc7b5677086 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Tue, 16 Jul 2024 14:53:26 +0300 Subject: [PATCH 013/206] remove debug repo name --- scripts/uninstall.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index babfd176..9e1f275c 100644 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -41,7 +41,6 @@ if $full; then fi rm -rf /usr/src/mytonctrl -rm -rf /usr/src/mytonctrl-v2 #todo remove rm -rf /usr/src/mtc-jsonrpc rm -rf /usr/src/pytonv3 rm -rf /tmp/myton* From c188aba6d7a94e585d8aa375c83d48468e374219 Mon Sep 17 00:00:00 2001 From: "dl@tonstakers.com" Date: Tue, 16 Jul 2024 14:54:44 +0300 Subject: [PATCH 014/206] remove readme tonstakers description --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 19879e13..53bdb472 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,6 @@ ![GitHub stars](https://img.shields.io/github/stars/ton-blockchain/mytonctrl?style=flat-square&logo=github) ![GitHub forks](https://img.shields.io/github/forks/ton-blockchain/mytonctrl?style=flat-square&logo=github) ![GitHub issues](https://img.shields.io/github/issues/ton-blockchain/mytonctrl?style=flat-square&logo=github) ![GitHub pull requests](https://img.shields.io/github/issues-pr/ton-blockchain/mytonctrl?style=flat-square&logo=github) ![GitHub last commit](https://img.shields.io/github/last-commit/ton-blockchain/mytonctrl?style=flat-square&logo=github) ![GitHub license](https://img.shields.io/github/license/ton-blockchain/mytonctrl?style=flat-square&logo=github) - -# Tonstakers -This fork focuses on supporting the second version of the mytonctrl utility, as well as testing installation script changes, which will later be submitted upstream. - + # MyTonCtrl [Данный текст доступен на русском языке.](README.Ru.md) From dc45cd27fc79093eb0e11851d7619cf518bcf291 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 13 Aug 2024 15:06:23 +0900 Subject: [PATCH 015/206] fix update/upgrade commit --- mytonctrl/mytonctrl.py | 7 +++++-- mytonctrl/scripts/upgrade.sh | 4 ++-- mytonctrl/utils.py | 8 ++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 33224717..53b8c158 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -42,7 +42,7 @@ ) from mytoncore.telemetry import is_host_virtual from mytonctrl.migrate import run_migrations -from mytonctrl.utils import GetItemFromList, timestamp2utcdatetime, fix_git_config +from mytonctrl.utils import GetItemFromList, timestamp2utcdatetime, fix_git_config, is_hex import sys, getopt, os @@ -334,8 +334,11 @@ def check_git(input_args, default_repo, text, default_branch='master'): #end define def check_branch_exists(author, repo, branch): + if is_hex(branch): + print('Hex name detected, skip branch existence check.') + return url = f"https://github.com/{author}/{repo}.git" - args = ["git", "ls-remote", "--heads", url, branch] + args = ["git", "ls-remote", "--heads", "--tags", url, branch] process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3) output = process.stdout.decode("utf-8") if branch not in output: diff --git a/mytonctrl/scripts/upgrade.sh b/mytonctrl/scripts/upgrade.sh index 7814f0aa..25a29632 100644 --- a/mytonctrl/scripts/upgrade.sh +++ b/mytonctrl/scripts/upgrade.sh @@ -63,8 +63,8 @@ rm -rf ${srcdir}/${repo} # Update code echo "https://github.com/${author}/${repo}.git -> ${branch}" -git clone --branch ${branch} --recursive https://github.com/${author}/${repo}.git -cd ${repo} +git clone --recursive https://github.com/${author}/${repo}.git +cd ${repo} && git checkout ${branch} export CC=/usr/bin/clang export CXX=/usr/bin/clang++ export CCACHE_DISABLE=1 diff --git a/mytonctrl/utils.py b/mytonctrl/utils.py index ba6cd105..7d6ab54d 100644 --- a/mytonctrl/utils.py +++ b/mytonctrl/utils.py @@ -15,6 +15,14 @@ def GetItemFromList(data, index): pass +def is_hex(s): + try: + int(s, 16) + return True + except ValueError: + return False + + def fix_git_config(git_path: str): args = ["git", "status"] try: From 21714a98eb7cd85a36c7b25685ee820d0c78c4b1 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 14 Aug 2024 13:28:03 +0900 Subject: [PATCH 016/206] rm domains --- README.md | 6 -- mytoncore/functions.py | 7 -- mytoncore/models.py | 9 -- mytoncore/mytoncore.py | 120 -------------------------- mytonctrl/mytonctrl.py | 134 +++-------------------------- mytonctrl/resources/translate.json | 20 ----- 6 files changed, 11 insertions(+), 285 deletions(-) diff --git a/README.md b/README.md index c577d8b4..4e9dc16a 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,6 @@ Mytonctrl's documentation can be found at https://docs.ton.org/participate/run-n - [x] Show offers - [x] Vote for the proposal - [x] Automatic voting for previously voted proposals -- [x] Domain management - - [x] Rent a new domain - - [x] Show rented domains - - [x] Show domain status - - [x] Delete domain - - [ ] Automatic domain renewal - [x] Controlling the validator - [x] Participate in the election of a validator - [x] Return bet + reward diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 0cca873a..1acec162 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -384,12 +384,6 @@ def Offers(local, ton): ton.VoteOffer(offer_hash) # end define - -def Domains(local, ton): - pass -# end define - - def Telemetry(local, ton): sendTelemetry = local.db.get("sendTelemetry") if sendTelemetry is not True: @@ -565,7 +559,6 @@ def General(local): local.start_cycle(Complaints, sec=t, args=(local, ton, )) local.start_cycle(Slashing, sec=t, args=(local, ton, )) - local.start_cycle(Domains, sec=600, args=(local, ton, )) local.start_cycle(Telemetry, sec=60, args=(local, ton, )) local.start_cycle(OverlayTelemetry, sec=7200, args=(local, ton, )) local.start_cycle(ScanLiteServers, sec=60, args=(local, ton,)) diff --git a/mytoncore/models.py b/mytoncore/models.py index 474fa276..db803d9b 100644 --- a/mytoncore/models.py +++ b/mytoncore/models.py @@ -41,15 +41,6 @@ def __init__(self, workchain, addr): #end class -class Domain(dict): - def __init__(self): - self["name"] = None - self["adnlAddr"] = None - self["walletName"] = None - #end define -#end class - - class Block(): def __init__(self, str=None): self.workchain = None diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index ed643083..e34baf14 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -19,7 +19,6 @@ from mytoncore.models import ( Wallet, Account, - Domain, Block, Trans, Message, @@ -390,48 +389,6 @@ def GetComment(self, body): return result #end define - def GetDomainAddr(self, domainName): - cmd = "dnsresolve {domainName} -1".format(domainName=domainName) - result = self.liteClient.Run(cmd) - if "not found" in result: - raise Exception("GetDomainAddr error: domain \"{domainName}\" not found".format(domainName=domainName)) - resolver = parse(result, "next resolver", '\n') - buff = resolver.replace(' ', '') - buffList = buff.split('=') - fullHexAddr = buffList[0] - addr = buffList[1] - return addr - #end define - - def GetDomainEndTime(self, domainName): - self.local.add_log("start GetDomainEndTime function", "debug") - buff = domainName.split('.') - subdomain = buff.pop(0) - dnsDomain = ".".join(buff) - dnsAddr = self.GetDomainAddr(dnsDomain) - - cmd = "runmethodfull {addr} getexpiration \"{subdomain}\"".format(addr=dnsAddr, subdomain=subdomain) - result = self.liteClient.Run(cmd) - result = parse(result, "result:", '\n') - result = parse(result, "[", "]") - result = result.replace(' ', '') - result = int(result) - return result - #end define - - def GetDomainAdnlAddr(self, domainName): - self.local.add_log("start GetDomainAdnlAddr function", "debug") - cmd = "dnsresolve {domainName} 1".format(domainName=domainName) - result = self.liteClient.Run(cmd) - lines = result.split('\n') - for line in lines: - if "adnl address" in line: - adnlAddr = parse(line, "=", "\n") - adnlAddr = adnlAddr.replace(' ', '') - adnlAddr = adnlAddr - return adnlAddr - #end define - def GetLocalWallet(self, walletName, version=None, subwallet=None): self.local.add_log("start GetLocalWallet function", "debug") if walletName is None: @@ -2898,76 +2855,6 @@ def GetItemFromDict(self, data, search): return None #end define - def GetDomainFromAuction(self, walletName, addr): - wallet = self.GetLocalWallet(walletName) - bocPath = self.local.buffer.my_temp_dir + "get_dns_data.boc" - bocData = bytes.fromhex("b5ee9c7241010101000e0000182fcb26a20000000000000000f36cae4d") - with open(bocPath, 'wb') as file: - file.write(bocData) - resultFilePath = self.SignBocWithWallet(wallet, bocPath, addr, 0.1) - self.SendFile(resultFilePath, wallet) - #end define - - def NewDomain(self, domain): - self.local.add_log("start NewDomain function", "debug") - domainName = domain["name"] - buff = domainName.split('.') - subdomain = buff.pop(0) - dnsDomain = ".".join(buff) - dnsAddr = self.GetDomainAddr(dnsDomain) - wallet = self.GetLocalWallet(domain["walletName"]) - expireInSec = 700000 # fix me - catId = 1 # fix me - - # Check if domain is busy - domainEndTime = self.GetDomainEndTime(domainName) - if domainEndTime > 0: - raise Exception("NewDomain error: domain is busy") - #end if - - fileName = self.tempDir + "dns-msg-body.boc" - args = ["auto-dns.fif", dnsAddr, "add", subdomain, expireInSec, "owner", wallet.addrB64, "cat", catId, "adnl", domain["adnlAddr"], "-o", fileName] - result = self.fift.Run(args) - resultFilePath = parse(result, "Saved to file ", ')') - resultFilePath = self.SignBocWithWallet(wallet, resultFilePath, dnsAddr, 1.7) - self.SendFile(resultFilePath, wallet) - self.AddDomain(domain) - #end define - - def AddDomain(self, domain): - if "domains" not in self.local.db: - self.local.db["domains"] = list() - #end if - self.local.db["domains"].append(domain) - self.local.save() - #end define - - def GetDomains(self): - domains = self.local.db.get("domains", list()) - for domain in domains: - domainName = domain.get("name") - domain["endTime"] = self.GetDomainEndTime(domainName) - return domains - #end define - - def GetDomain(self, domainName): - domain = dict() - domain["name"] = domainName - domain["adnlAddr"] = self.GetDomainAdnlAddr(domainName) - domain["endTime"] = self.GetDomainEndTime(domainName) - return domain - #end define - - def DeleteDomain(self, domainName): - domains = self.local.db.get("domains") - for domain in domains: - if (domainName == domain.get("name")): - domains.remove(domain) - self.local.save() - return - raise Exception("DeleteDomain error: Domain not found") - #end define - def GetAutoTransferRules(self): autoTransferRules = self.local.db.get("autoTransferRules") if autoTransferRules is None: @@ -3030,13 +2917,6 @@ def WriteBookmarkData(self, bookmark): data = "empty" else: data = account.balance - elif type == "domain": - domainName = bookmark.get("addr") - endTime = self.GetDomainEndTime(domainName) - if endTime == 0: - data = "free" - else: - data = timestamp2datetime(endTime, "%d.%m.%Y") else: data = "null" bookmark["data"] = data diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 33224717..9b04fca6 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -101,16 +101,6 @@ def inject_globals(func): console.AddItem("bl", inject_globals(PrintBookmarksList), local.translate("bl_cmd")) console.AddItem("db", inject_globals(DeleteBookmark), local.translate("db_cmd")) - # console.AddItem("nr", inject_globals(CreatNewAutoTransferRule), local.translate("nr_cmd")) # "Добавить правило автопереводов в расписание / Create new auto transfer rule" - # console.AddItem("rl", inject_globals(PrintAutoTransferRulesList), local.translate("rl_cmd")) # "Показать правила автопереводов / Show auto transfer rule list" - # console.AddItem("dr", inject_globals(DeleteAutoTransferRule), local.translate("dr_cmd")) # "Удалить правило автопереводов из расписания / Delete auto transfer rule" - - # console.AddItem("nd", inject_globals(NewDomain), local.translate("nd_cmd")) - # console.AddItem("dl", inject_globals(PrintDomainsList), local.translate("dl_cmd")) - # console.AddItem("vds", inject_globals(ViewDomainStatus), local.translate("vds_cmd")) - # console.AddItem("dd", inject_globals(DeleteDomain), local.translate("dd_cmd")) - # console.AddItem("gdfa", inject_globals(GetDomainFromAuction), local.translate("gdfa_cmd")) - console.AddItem("ol", inject_globals(PrintOffersList), local.translate("ol_cmd")) console.AddItem("od", inject_globals(OfferDiff), local.translate("od_cmd")) @@ -315,7 +305,7 @@ def check_git(input_args, default_repo, text, default_branch='master'): need_branch = data.get("branch") # Check if remote repo is different from default - if ((need_author is None and local_author != default_author) or + if ((need_author is None and local_author != default_author) or (need_repo is None and local_repo != default_repo)): remote_url = f"https://github.com/{local_author}/{local_repo}/tree/{need_branch if need_branch else local_branch}" raise Exception(f"{text} error: You are on {remote_url} remote url, to update to the tip use `{text} {remote_url}` command") @@ -575,7 +565,7 @@ def PrintStatus(local, ton, args): config34 = ton.GetConfig34() config36 = ton.GetConfig36() totalValidators = config34["totalValidators"] - + if opt != "fast": onlineValidators = ton.GetOnlineValidators() validator_efficiency = ton.GetValidatorEfficiency() @@ -586,26 +576,26 @@ def PrintStatus(local, ton, args): if oldStartWorkTime is None: oldStartWorkTime = config34.get("startWorkTime") shardsNumber = ton.GetShardsNumber() - + config15 = ton.GetConfig15() config17 = ton.GetConfig17() fullConfigAddr = ton.GetFullConfigAddr() fullElectorAddr = ton.GetFullElectorAddr() startWorkTime = ton.GetActiveElectionId(fullElectorAddr) validator_index = ton.GetValidatorIndex() - + offersNumber = ton.GetOffersNumber() complaintsNumber = ton.GetComplaintsNumber() - + tpsAvg = ton.GetStatistics("tpsAvg", statistics) - + if validator_wallet is not None: validator_account = ton.GetAccount(validator_wallet.addrB64) #end if if all_status: PrintTonStatus(local, network_name, startWorkTime, totalValidators, onlineValidators, shardsNumber, offersNumber, complaintsNumber, tpsAvg) - PrintLocalStatus(local, adnl_addr, validator_index, validator_efficiency, validator_wallet, validator_account, validator_status, + PrintLocalStatus(local, adnl_addr, validator_index, validator_efficiency, validator_wallet, validator_account, validator_status, db_size, db_usage, memory_info, swap_info, net_load_avg, disks_load_avg, disks_load_percent_avg, fullnode_adnl) if all_status: PrintTonConfig(local, fullConfigAddr, fullElectorAddr, config15, config17) @@ -741,7 +731,7 @@ def PrintLocalStatus(local, adnlAddr, validatorIndex, validatorEfficiency, valid dbSize_text = GetColorInt(dbSize, 1000, logic="less", ending=" Gb") dbUsage_text = GetColorInt(dbUsage, 80, logic="less", ending="%") dbStatus_text = local.translate("local_status_db").format(dbSize_text, dbUsage_text) - + # Mytonctrl and validator git hash mtcGitPath = "/usr/src/mytonctrl" validatorGitPath = "/usr/src/ton" @@ -769,7 +759,7 @@ def PrintLocalStatus(local, adnlAddr, validatorIndex, validatorEfficiency, valid print(cpuLoad_text) print(netLoad_text) print(memoryLoad_text) - + print(disksLoad_text) print(mytoncoreStatus_text) print(validatorStatus_text) @@ -1087,12 +1077,10 @@ def CreatNewBookmark(ton, args): name = args[0] addr = args[1] except: - color_print("{red}Bad args. Usage:{endc} nb ") + color_print("{red}Bad args. Usage:{endc} nb ") return if ton.IsAddr(addr): type = "account" - else: - type = "domain" #end if bookmark = dict() @@ -1109,7 +1097,7 @@ def PrintBookmarksList(ton, args): print("No data") return table = list() - table += [["Name", "Type", "Address / Domain", "Balance / Exp. date"]] + table += [["Name", "Type", "Address", "Balance / Exp. date"]] for item in data: name = item.get("name") type = item.get("type") @@ -1130,36 +1118,6 @@ def DeleteBookmark(ton, args): color_print("DeleteBookmark - {green}OK{endc}") #end define -# def CreatNewAutoTransferRule(args): -# try: -# name = args[0] -# addr = args[1] -# except: -# color_print("{red}Bad args. Usage:{endc} nr ") -# return -# rule = dict() -# rule["name"] = name -# rule["addr"] = addr -# ton.AddAutoTransferRule(rule) -# color_print("CreatNewAutoTransferRule - {green}OK{endc}") -# #end define - -# def PrintAutoTransferRulesList(args): -# data = ton.GetRules() -# if (data is None or len(data) == 0): -# print("No data") -# return -# table = list() -# table += [["Name", "fix me"]] -# for item in data: -# table += [[item.get("name"), item.get("fix me")]] -# print_table(table) -# #end define - -# def DeleteAutoTransferRule(args): -# print("fix me") -# #end define - def PrintOffersList(ton, args): data = ton.GetOffers() if (data is None or len(data) == 0): @@ -1244,76 +1202,6 @@ def PrintComplaintsList(ton, args): print_table(table) #end define -def NewDomain(ton, args): - try: - domainName = args[0] - walletName = args[1] - adnlAddr = args[2] - except: - color_print("{red}Bad args. Usage:{endc} nd ") - return - domain = dict() - domain["name"] = domainName - domain["adnlAddr"] = adnlAddr - domain["walletName"] = walletName - ton.NewDomain(domain) - color_print("NewDomain - {green}OK{endc}") -#end define - -def PrintDomainsList(ton, args): - data = ton.GetDomains() - if (data is None or len(data) == 0): - print("No data") - return - table = list() - table += [["Domain", "Wallet", "Expiration date", "ADNL address"]] - for item in data: - domainName = item.get("name") - walletName = item.get("walletName") - endTime = item.get("endTime") - endTime = timestamp2datetime(endTime, "%d.%m.%Y") - adnlAddr = item.get("adnlAddr") - table += [[domainName, walletName, endTime, adnlAddr]] - print_table(table) -#end define - -def ViewDomainStatus(ton, args): - try: - domainName = args[0] - except: - color_print("{red}Bad args. Usage:{endc} vds ") - return - domain = ton.GetDomain(domainName) - endTime = domain.get("endTime") - endTime = timestamp2datetime(endTime, "%d.%m.%Y") - adnlAddr = domain.get("adnlAddr") - table = list() - table += [["Domain", "Expiration date", "ADNL address"]] - table += [[domainName, endTime, adnlAddr]] - print_table(table) -#end define - -def DeleteDomain(ton, args): - try: - domainName = args[0] - except: - color_print("{red}Bad args. Usage:{endc} dd ") - return - ton.DeleteDomain(domainName) - color_print("DeleteDomain - {green}OK{endc}") -#end define - -def GetDomainFromAuction(ton, args): - try: - walletName = args[0] - addr = args[1] - except: - color_print("{red}Bad args. Usage:{endc} gdfa ") - return - ton.GetDomainFromAuction(walletName, addr) - color_print("GetDomainFromAuction - {green}OK{endc}") -#end define - def PrintElectionEntriesList(ton, args): past = "past" in args data = ton.GetElectionEntries(past=past) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 7bbfcf1d..2caf4ec8 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -129,26 +129,6 @@ "ru": "Удалить закладку", "zh_TW": "刪除書籤" }, - "nd_cmd": { - "en": "New domain", - "ru": "Арендовать новый домен", - "zh_TW": "新建域名" - }, - "dl_cmd": { - "en": "Show domain list", - "ru": "Показать арендованные домены", - "zh_TW": "顯示域名列表" - }, - "vds_cmd": { - "en": "View domain status", - "ru": "Показать статус домена", - "zh_TW": "查看域名狀態" - }, - "dd_cmd": { - "en": "Delete domain", - "ru": "Удалить домен", - "zh_TW": "刪除域名" - }, "ol_cmd": { "en": "Show offers list", "ru": "Показать действующие предложения", From 3884b46bd6704599257b581aed4fb57109c21f22 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 15 Aug 2024 14:12:51 +0900 Subject: [PATCH 017/206] refactor mytonctrl.py: add wallet and utilites modules --- modules/telemetry.py | 152 +++++++++++++ modules/utilities.py | 412 ++++++++++++++++++++++++++++++++++++ modules/wallet.py | 161 ++++++++++++++ mytoncore/mytoncore.py | 140 +----------- mytonctrl/mytonctrl.py | 470 ++--------------------------------------- 5 files changed, 738 insertions(+), 597 deletions(-) create mode 100644 modules/telemetry.py create mode 100644 modules/utilities.py create mode 100644 modules/wallet.py diff --git a/modules/telemetry.py b/modules/telemetry.py new file mode 100644 index 00000000..e1441b59 --- /dev/null +++ b/modules/telemetry.py @@ -0,0 +1,152 @@ +import os +import subprocess + +import psutil + +from modules.module import MtcModule +from mytoncore.utils import parse_db_stats +from mypylib.mypylib import get_service_pid + + +class TelemetryModule(MtcModule): + + @staticmethod + def GetUname(): + data = os.uname() + result = dict( + zip('sysname nodename release version machine'.split(), data)) + result.pop("nodename") + return result + # end define + + @staticmethod + def GetMemoryInfo(): + result = dict() + data = psutil.virtual_memory() + result["total"] = round(data.total / 10 ** 9, 2) + result["usage"] = round(data.used / 10 ** 9, 2) + result["usagePercent"] = data.percent + return result + # end define + + @staticmethod + def GetSwapInfo(): + result = dict() + data = psutil.swap_memory() + result["total"] = round(data.total / 10 ** 9, 2) + result["usage"] = round(data.used / 10 ** 9, 2) + result["usagePercent"] = data.percent + return result + # end define + + @staticmethod + def GetValidatorProcessInfo(): + pid = get_service_pid("validator") + if pid == None or pid == 0: + return + p = psutil.Process(pid) + mem = p.memory_info() + result = dict() + result["cpuPercent"] = p.cpu_percent() + memory = dict() + memory["rss"] = mem.rss + memory["vms"] = mem.vms + memory["shared"] = mem.shared + memory["text"] = mem.text + memory["lib"] = mem.lib + memory["data"] = mem.data + memory["dirty"] = mem.dirty + result["memory"] = memory + # io = p.io_counters() # Permission denied: '/proc/{pid}/io' + return result + # end define + + @staticmethod + def get_db_stats(): + result = { + 'rocksdb': { + 'ok': True, + 'message': '', + 'data': {} + }, + 'celldb': { + 'ok': True, + 'message': '', + 'data': {} + }, + } + rocksdb_stats_path = '/var/ton-work/db/db_stats.txt' + celldb_stats_path = '/var/ton-work/db/celldb/db_stats.txt' + if os.path.exists(rocksdb_stats_path): + try: + result['rocksdb']['data'] = parse_db_stats(rocksdb_stats_path) + except Exception as e: + result['rocksdb']['ok'] = False + result['rocksdb']['message'] = f'failed to fetch db stats: {e}' + else: + result['rocksdb']['ok'] = False + result['rocksdb']['message'] = 'db stats file is not exists' + # end if + + if os.path.exists(celldb_stats_path): + try: + result['celldb']['data'] = parse_db_stats(celldb_stats_path) + except Exception as e: + result['celldb']['ok'] = False + result['celldb']['message'] = f'failed to fetch db stats: {e}' + else: + result['celldb']['ok'] = False + result['celldb']['message'] = 'db stats file is not exists' + # end if + return result + # end define + + @staticmethod + def get_cpu_name(): + with open('/proc/cpuinfo') as f: + for line in f: + if line.strip(): + if line.rstrip('\n').startswith('model name'): + return line.rstrip('\n').split(':')[1].strip() + return None + # end define + + @staticmethod + def is_host_virtual(): + try: + with open('/sys/class/dmi/id/product_name') as f: + product_name = f.read().strip().lower() + if 'virtual' in product_name or 'kvm' in product_name or 'qemu' in product_name or 'vmware' in product_name: + return {'virtual': True, 'product_name': product_name} + return {'virtual': False, 'product_name': product_name} + except FileNotFoundError: + return {'virtual': None, 'product_name': None} + # end define + + @staticmethod + def do_beacon_ping(host, count, timeout): + args = ['ping', '-c', str(count), '-W', str(timeout), host] + process = subprocess.run(args, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + output = process.stdout.decode("utf-8") + avg = output.split('\n')[-2].split('=')[1].split('/')[1] + return float(avg) + # end define + + @classmethod + def get_pings_values(cls): + return { + 'beacon-eu-01.toncenter.com': cls.do_beacon_ping('beacon-eu-01.toncenter.com', 5, 10), + 'beacon-apac-01.toncenter.com': cls.do_beacon_ping('beacon-apac-01.toncenter.com', 5, 10) + } + # end define + + @staticmethod + def get_validator_disk_name(): + process = subprocess.run("df -h /var/ton-work/ | sed -n '2 p' | awk '{print $1}'", stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3, shell=True) + output = process.stdout.decode("utf-8") + return output.strip() + # end define + + def add_console_commands(self, console): ... diff --git a/modules/utilities.py b/modules/utilities.py new file mode 100644 index 00000000..49ff82c7 --- /dev/null +++ b/modules/utilities.py @@ -0,0 +1,412 @@ +import base64 +import json +import os +import subprocess +import time + +from mypylib.mypylib import color_print, print_table, color_text, timeago, bcolors +from modules.module import MtcModule + + +class UtilitiesModule(MtcModule): + + description = '' + default_value = False + + def view_account_status(self, args): + try: + addrB64 = args[0] + except: + color_print("{red}Bad args. Usage:{endc} vas ") + return + addrB64 = self.get_destination_addr(addrB64) + account = self.ton.GetAccount(addrB64) + version = self.ton.GetVersionFromCodeHash(account.codeHash) + statusTable = list() + statusTable += [["Address", "Status", "Balance", "Version"]] + statusTable += [[addrB64, account.status, account.balance, version]] + codeHashTable = list() + codeHashTable += [["Code hash"]] + codeHashTable += [[account.codeHash]] + historyTable = self.get_history_table(addrB64, 10) + print_table(statusTable) + print() + print_table(codeHashTable) + print() + print_table(historyTable) + + # end define + + def get_history_table(self, addr, limit): + addr = self.get_destination_addr(addr) + account = self.ton.GetAccount(addr) + history = self.ton.GetAccountHistory(account, limit) + table = list() + typeText = color_text("{red}{bold}{endc}") + table += [["Time", typeText, "Coins", "From/To"]] + for message in history: + if message.srcAddr is None: + continue + srcAddrFull = f"{message.srcWorkchain}:{message.srcAddr}" + destAddFull = f"{message.destWorkchain}:{message.destAddr}" + if srcAddrFull == account.addrFull: + type = color_text("{red}{bold}>>>{endc}") + fromto = destAddFull + else: + type = color_text("{blue}{bold}<<<{endc}") + fromto = srcAddrFull + fromto = self.ton.AddrFull2AddrB64(fromto) + # datetime = timestamp2datetime(message.time, "%Y.%m.%d %H:%M:%S") + datetime = timeago(message.time) + table += [[datetime, type, message.value, fromto]] + return table + # end define + + def view_account_history(self, args): + try: + addr = args[0] + limit = int(args[1]) + except: + color_print("{red}Bad args. Usage:{endc} vah ") + return + table = self.get_history_table(addr, limit) + print_table(table) + # end define + + def get_destination_addr(self, destination): + if self.ton.IsAddrB64(destination): + pass + elif self.ton.IsAddrFull(destination): + destination = self.ton.AddrFull2AddrB64(destination) + else: + wallets_name_list = self.ton.GetWalletsNameList() + if destination in wallets_name_list: + wallet = self.ton.GetLocalWallet(destination) + destination = wallet.addrB64 + return destination + # end define + + def move_coins(self, args): + try: + wallet_name = args[0] + destination = args[1] + amount = args[2] + flags = args[3:] + except: + color_print("{red}Bad args. Usage:{endc} mg ") + return + wallet = self.ton.GetLocalWallet(wallet_name) + destination = self.get_destination_addr(destination) + self.ton.MoveCoins(wallet, destination, amount, flags=flags) + color_print("MoveCoins - {green}OK{endc}") + # end define + + def do_move_coins_through_proxy(self, wallet, dest, coins): + self.local.add_log("start MoveCoinsThroughProxy function", "debug") + wallet1 = self.ton.CreateWallet("proxy_wallet1", 0) + wallet2 = self.ton.CreateWallet("proxy_wallet2", 0) + self.ton.MoveCoins(wallet, wallet1.addrB64_init, coins) + self.ton.ActivateWallet(wallet1) + self.ton.MoveCoins(wallet1, wallet2.addrB64_init, "alld") + self.ton.ActivateWallet(wallet2) + self.ton.MoveCoins(wallet2, dest, "alld", flags=["-n"]) + wallet1.Delete() + wallet2.Delete() + # end define + + def move_coins_through_proxy(self, args): + try: + wallet_name = args[0] + destination = args[1] + amount = args[2] + except: + color_print("{red}Bad args. Usage:{endc} mgtp ") + return + wallet = self.ton.GetLocalWallet(wallet_name) + destination = self.get_destination_addr(destination) + self.do_move_coins_through_proxy(wallet, destination, amount) + color_print("MoveCoinsThroughProxy - {green}OK{endc}") + # end define + + def create_new_bookmark(self, args): + try: + name = args[0] + addr = args[1] + except: + color_print("{red}Bad args. Usage:{endc} nb ") + return + if self.ton.IsAddr(addr): + type = "account" + # end if + + bookmark = dict() + bookmark["name"] = name + bookmark["type"] = type + bookmark["addr"] = addr + self.ton.AddBookmark(bookmark) + color_print("CreatNewBookmark - {green}OK{endc}") + # end define + + def print_bookmarks_list(self, args): + data = self.ton.GetBookmarks() + if data is None or len(data) == 0: + print("No data") + return + table = list() + table += [["Name", "Type", "Address", "Balance / Exp. date"]] + for item in data: + name = item.get("name") + type = item.get("type") + addr = item.get("addr") + bookmark_data = item.get("data") + table += [[name, type, addr, bookmark_data]] + print_table(table) + # end define + + def delete_bookmark(self, args): + try: + name = args[0] + type = args[1] + except: + color_print("{red}Bad args. Usage:{endc} db ") + return + self.ton.DeleteBookmark(name, type) + color_print("DeleteBookmark - {green}OK{endc}") + # end define + + @staticmethod + def reduct(item): + item = str(item) + if item is None: + result = None + else: + end = len(item) + result = item[0:6] + "..." + item[end - 6:end] + return result + # end define + + def print_offers_list(self, args): + data = self.ton.GetOffers() + if data is None or len(data) == 0: + print("No data") + return + if "--json" in args: + text = json.dumps(data, indent=2) + print(text) + else: + table = list() + table += [["Hash", "Config", "Votes", "W/L", "Approved", "Is passed"]] + for item in data: + hash = item.get("hash") + votedValidators = len(item.get("votedValidators")) + wins = item.get("wins") + losses = item.get("losses") + wl = "{0}/{1}".format(wins, losses) + approvedPercent = item.get("approvedPercent") + approvedPercent_text = "{0}%".format(approvedPercent) + isPassed = item.get("isPassed") + if "hash" not in args: + hash = self.reduct(hash) + if isPassed == True: + isPassed = bcolors.green_text("true") + if isPassed == False: + isPassed = bcolors.red_text("false") + table += [[hash, item.config.id, votedValidators, wl, approvedPercent_text, isPassed]] + print_table(table) + # end define + + def get_offer_diff(self, offer_hash): + self.local.add_log("start GetOfferDiff function", "debug") + offer = self.ton.GetOffer(offer_hash) + config_id = offer["config"]["id"] + config_value = offer["config"]["value"] + + if '{' in config_value or '}' in config_value: + start = config_value.find('{') + 1 + end = config_value.find('}') + config_value = config_value[start:end] + # end if + + args = [self.ton.liteClient.appPath, "--global-config", self.ton.liteClient.configPath, "--verbosity", "0"] + process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + time.sleep(1) + + fullConfigAddr = self.ton.GetFullConfigAddr() + cmd = "runmethodfull {fullConfigAddr} list_proposals".format(fullConfigAddr=fullConfigAddr) + process.stdin.write(cmd.encode() + b'\n') + process.stdin.flush() + time.sleep(1) + + cmd = "dumpcellas ConfigParam{configId} {configValue}".format(configId=config_id, configValue=config_value) + process.stdin.write(cmd.encode() + b'\n') + process.stdin.flush() + time.sleep(1) + + process.terminate() + text = process.stdout.read().decode() + + lines = text.split('\n') + b = len(lines) + for i in range(b): + line = lines[i] + if "dumping cells as values of TLB type" in line: + a = i + 2 + break + # end for + + for i in range(a, b): + line = lines[i] + if '(' in line: + start = i + break + # end for + + for i in range(a, b): + line = lines[i] + if '>' in line: + end = i + break + # end for + + buff = lines[start:end] + text = "".join(buff) + newData = self.ton.Tlb2Json(text) + newFileName = self.ton.tempDir + "data1diff" + file = open(newFileName, 'wt') + newText = json.dumps(newData, indent=2) + file.write(newText) + file.close() + + oldData = self.ton.GetConfig(config_id) + oldFileName = self.ton.tempDir + "data2diff" + file = open(oldFileName, 'wt') + oldText = json.dumps(oldData, indent=2) + file.write(oldText) + file.close() + + print(oldText) + args = ["diff", "--color", oldFileName, newFileName] + subprocess.run(args) + # end define + + def offer_diff(self, args): + try: + offer_hash = args[0] + offer_hash = offer_hash + except: + color_print("{red}Bad args. Usage:{endc} od ") + return + self.get_offer_diff(offer_hash) + # end define + + def print_complaints_list(self, args): + past = "past" in args + data = self.ton.GetComplaints(past=past) + if data is None or len(data) == 0: + print("No data") + return + if "--json" in args: + text = json.dumps(data, indent=2) + print(text) + else: + table = list() + table += [["Election id", "ADNL", "Fine (part)", "Votes", "Approved", "Is passed"]] + for key, item in data.items(): + electionId = item.get("electionId") + adnl = item.get("adnl") + suggestedFine = item.get("suggestedFine") + suggestedFinePart = item.get("suggestedFinePart") + Fine_text = "{0} ({1})".format(suggestedFine, suggestedFinePart) + votedValidators = len(item.get("votedValidators")) + approvedPercent = item.get("approvedPercent") + approvedPercent_text = "{0}%".format(approvedPercent) + isPassed = item.get("isPassed") + if "adnl" not in args: + adnl = self.reduct(adnl) + if isPassed: + isPassed = bcolors.green_text("true") + if not isPassed: + isPassed = bcolors.red_text("false") + table += [[electionId, adnl, Fine_text, votedValidators, approvedPercent_text, isPassed]] + print_table(table) + # end define + + def print_election_entries_list(self, args): + past = "past" in args + data = self.ton.GetElectionEntries(past=past) + if data is None or len(data) == 0: + print("No data") + return + if "--json" in args: + text = json.dumps(data, indent=2) + print(text) + else: + table = list() + table += [["ADNL", "Pubkey", "Wallet", "Stake", "Max-factor"]] + for key, item in data.items(): + adnl = item.get("adnlAddr") + pubkey = item.get("pubkey") + walletAddr = item.get("walletAddr") + stake = item.get("stake") + maxFactor = item.get("maxFactor") + if "adnl" not in args: + adnl = self.reduct(adnl) + if "pubkey" not in args: + pubkey = self.reduct(pubkey) + if "wallet" not in args: + walletAddr = self.reduct(walletAddr) + table += [[adnl, pubkey, walletAddr, stake, maxFactor]] + print_table(table) + # end define + + def print_validator_list(self, args): + past = "past" in args + data = self.ton.GetValidatorsList(past=past) + if data is None or len(data) == 0: + print("No data") + return + if "--json" in args: + text = json.dumps(data, indent=2) + print(text) + else: + table = list() + table += [["ADNL", "Pubkey", "Wallet", "Efficiency", "Online"]] + for item in data: + adnl = item.get("adnlAddr") + pubkey = item.get("pubkey") + walletAddr = item.get("walletAddr") + efficiency = item.get("efficiency") + online = item.get("online") + if "adnl" not in args: + adnl = self.reduct(adnl) + if "pubkey" not in args: + pubkey = self.reduct(pubkey) + if "wallet" not in args: + walletAddr = self.reduct(walletAddr) + if "offline" in args and online != False: + continue + if online: + online = bcolors.green_text("true") + if not online: + online = bcolors.red_text("false") + table += [[adnl, pubkey, walletAddr, efficiency, online]] + print_table(table) + # end define + + def add_console_commands(self, console): + console.AddItem("vas", self.view_account_status, self.local.translate("vas_cmd")) + console.AddItem("vah", self.view_account_history, self.local.translate("vah_cmd")) + console.AddItem("mg", self.move_coins, self.local.translate("mg_cmd")) + console.AddItem("mgtp", self.move_coins_through_proxy, self.local.translate("mgtp_cmd")) + + console.AddItem("nb", self.create_new_bookmark, self.local.translate("nb_cmd")) + console.AddItem("bl", self.print_bookmarks_list, self.local.translate("bl_cmd")) + console.AddItem("db", self.delete_bookmark, self.local.translate("db_cmd")) + + console.AddItem("ol", self.print_offers_list, self.local.translate("ol_cmd")) + console.AddItem("od", self.offer_diff, self.local.translate("od_cmd")) + + console.AddItem("el", self.print_election_entries_list, self.local.translate("el_cmd")) + console.AddItem("vl", self.print_validator_list, self.local.translate("vl_cmd")) + console.AddItem("cl", self.print_complaints_list, self.local.translate("cl_cmd")) + diff --git a/modules/wallet.py b/modules/wallet.py new file mode 100644 index 00000000..d4b385c0 --- /dev/null +++ b/modules/wallet.py @@ -0,0 +1,161 @@ +import base64 +import os + +from modules.module import MtcModule +from mypylib.mypylib import color_print, print_table + + +class WalletModule(MtcModule): + + description = '' + default_value = False + + def create_new_wallet(self, args): + version = "v1" + try: + if len(args) == 0: + walletName = self.ton.GenerateWalletName() + workchain = 0 + else: + workchain = int(args[0]) + walletName = args[1] + if len(args) > 2: + version = args[2] + if len(args) == 4: + subwallet = int(args[3]) + else: + subwallet = 698983191 + workchain # 0x29A9A317 + workchain + except: + color_print("{red}Bad args. Usage:{endc} nw [ ]") + return + wallet = self.ton.CreateWallet(walletName, workchain, version, subwallet=subwallet) + table = list() + table += [["Name", "Workchain", "Address"]] + table += [[wallet.name, wallet.workchain, wallet.addrB64_init]] + print_table(table) + # end define + + def _wallets_check(self): + self.local.add_log("start WalletsCheck function", "debug") + wallets = self.get_wallets() + for wallet in wallets: + if os.path.isfile(wallet.bocFilePath): + account = self.ton.GetAccount(wallet.addrB64) + if account.balance > 0: + self.ton.SendFile(wallet.bocFilePath, wallet) + # end define + + def activate_wallet(self, args): + try: + walletName = args[0] + except Exception as err: + walletName = "all" + if walletName == "all": + self._wallets_check() + else: + wallet = self.ton.GetLocalWallet(walletName) + self.ton.ActivateWallet(wallet) + color_print("ActivateWallet - {green}OK{endc}") + # end define + + def get_wallets(self): + self.local.add_log("start GetWallets function", "debug") + wallets = list() + wallets_name_list = self.ton.GetWalletsNameList() + for walletName in wallets_name_list: + wallet = self.ton.GetLocalWallet(walletName) + wallets.append(wallet) + return wallets + # end define + + def print_wallets_list(self, args): + table = list() + table += [["Name", "Status", "Balance", "Ver", "Wch", "Address"]] + data = self.get_wallets() + if data is None or len(data) == 0: + print("No data") + return + for wallet in data: + account = self.ton.GetAccount(wallet.addrB64) + if account.status != "active": + wallet.addrB64 = wallet.addrB64_init + table += [[wallet.name, account.status, account.balance, wallet.version, wallet.workchain, wallet.addrB64]] + print_table(table) + # end define + + def do_import_wallet(self, addr_b64, key): + addr_bytes = self.ton.addr_b64_to_bytes(addr_b64) + pk_bytes = base64.b64decode(key) + wallet_name = self.ton.GenerateWalletName() + wallet_path = self.ton.walletsDir + wallet_name + with open(wallet_path + ".addr", 'wb') as file: + file.write(addr_bytes) + with open(wallet_path + ".pk", 'wb') as file: + file.write(pk_bytes) + return wallet_name + # end define + + def import_wallet(self, args): + try: + addr = args[0] + key = args[1] + except: + color_print("{red}Bad args. Usage:{endc} iw ") + return + name = self.do_import_wallet(addr, key) + print("Wallet name:", name) + # end define + + def set_wallet_version(self, args): + try: + addr = args[0] + version = args[1] + except: + color_print("{red}Bad args. Usage:{endc} swv ") + return + self.ton.SetWalletVersion(addr, version) + color_print("SetWalletVersion - {green}OK{endc}") + # end define + + def do_export_wallet(self, wallet_name): + wallet = self.ton.GetLocalWallet(wallet_name) + with open(wallet.privFilePath, 'rb') as file: + data = file.read() + key = base64.b64encode(data).decode("utf-8") + return wallet.addrB64, key + # end define + + def export_wallet(self, args): + try: + name = args[0] + except: + color_print("{red}Bad args. Usage:{endc} ew ") + return + addr, key = self.ton.ExportWallet(name) + print("Wallet name:", name) + print("Address:", addr) + print("Secret key:", key) + # end define + + def delete_wallet(self, args): + try: + wallet_name = args[0] + except: + color_print("{red}Bad args. Usage:{endc} dw ") + return + if input("Are you sure you want to delete this wallet (yes/no): ") != "yes": + print("Cancel wallet deletion") + return + wallet = self.ton.GetLocalWallet(wallet_name) + wallet.Delete() + color_print("DeleteWallet - {green}OK{endc}") + # end define + + def add_console_commands(self, console): + console.AddItem("nw", self.create_new_wallet, self.local.translate("nw_cmd")) + console.AddItem("aw", self.activate_wallet, self.local.translate("aw_cmd")) + console.AddItem("wl", self.print_wallets_list, self.local.translate("wl_cmd")) + console.AddItem("iw", self.import_wallet, self.local.translate("iw_cmd")) + console.AddItem("swv", self.set_wallet_version, self.local.translate("swv_cmd")) + console.AddItem("ew", self.export_wallet, self.local.translate("ex_cmd")) + console.AddItem("dw", self.delete_wallet, self.local.translate("dw_cmd")) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e34baf14..f1b03c8c 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1660,18 +1660,6 @@ def ActivateWallet(self, wallet): self.SendFile(wallet.bocFilePath, wallet, remove=False) #end define - def ImportWallet(self, addr_b64, key): - addr_bytes = self.addr_b64_to_bytes(addr_b64) - pk_bytes = base64.b64decode(key) - wallet_name = self.GenerateWalletName() - wallet_path = self.walletsDir + wallet_name - with open(wallet_path + ".addr", 'wb') as file: - file.write(addr_bytes) - with open(wallet_path + ".pk", 'wb') as file: - file.write(pk_bytes) - return wallet_name - #end define - def import_wallet_with_version(self, key, version, **kwargs): wallet_name = kwargs.get("wallet_name") workchain = kwargs.get("workchain", 0) @@ -1723,14 +1711,6 @@ def addr_b64_to_bytes(self, addr_b64): return result #end define - def ExportWallet(self, walletName): - wallet = self.GetLocalWallet(walletName) - with open(wallet.privFilePath, 'rb') as file: - data = file.read() - key = base64.b64encode(data).decode("utf-8") - return wallet.addrB64, key - #end define - def GetWalletsNameList(self): self.local.add_log("start GetWalletsNameList function", "debug") walletsNameList = list() @@ -1744,16 +1724,6 @@ def GetWalletsNameList(self): return walletsNameList #end define - def GetWallets(self): - self.local.add_log("start GetWallets function", "debug") - wallets = list() - walletsNameList = self.GetWalletsNameList() - for walletName in walletsNameList: - wallet = self.GetLocalWallet(walletName) - wallets.append(wallet) - return wallets - #end define - def GenerateWalletName(self): self.local.add_log("start GenerateWalletName function", "debug") index = 1 @@ -1776,16 +1746,6 @@ def GenerateWalletName(self): return walletName #end define - def WalletsCheck(self): - self.local.add_log("start WalletsCheck function", "debug") - wallets = self.GetWallets() - for wallet in wallets: - if os.path.isfile(wallet.bocFilePath): - account = self.GetAccount(wallet.addrB64) - if account.balance > 0: - self.SendFile(wallet.bocFilePath, wallet) - #end define - def GetValidatorConfig(self): #self.local.add_log("start GetValidatorConfig function", "debug") result = self.validatorConsole.Run("getconfig") @@ -1892,18 +1852,7 @@ def MoveCoins(self, wallet, dest, coins, **kwargs): self.SendFile(savedFilePath, wallet, timeout=timeout) #end define - def MoveCoinsThroughProxy(self, wallet, dest, coins): - self.local.add_log("start MoveCoinsThroughProxy function", "debug") - wallet1 = self.CreateWallet("proxy_wallet1", 0) - wallet2 = self.CreateWallet("proxy_wallet2", 0) - self.MoveCoins(wallet, wallet1.addrB64_init, coins) - self.ActivateWallet(wallet1) - self.MoveCoins(wallet1, wallet2.addrB64_init, "alld") - self.ActivateWallet(wallet2) - self.MoveCoins(wallet2, dest, "alld", flags=["-n"]) - wallet1.Delete() - wallet2.Delete() - #end define + def MoveCoinsFromHW(self, wallet, destList, **kwargs): self.local.add_log("start MoveCoinsFromHW function", "debug") @@ -2095,80 +2044,6 @@ def GetOffers(self): return offers #end define - def GetOfferDiff(self, offerHash): - self.local.add_log("start GetOfferDiff function", "debug") - offer = self.GetOffer(offerHash) - configId = offer["config"]["id"] - configValue = offer["config"]["value"] - - if '{' in configValue or '}' in configValue: - start = configValue.find('{') + 1 - end = configValue.find('}') - configValue = configValue[start:end] - #end if - - args = [self.liteClient.appPath, "--global-config", self.liteClient.configPath, "--verbosity", "0"] - process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - time.sleep(1) - - fullConfigAddr = self.GetFullConfigAddr() - cmd = "runmethodfull {fullConfigAddr} list_proposals".format(fullConfigAddr=fullConfigAddr) - process.stdin.write(cmd.encode() + b'\n') - process.stdin.flush() - time.sleep(1) - - cmd = "dumpcellas ConfigParam{configId} {configValue}".format(configId=configId, configValue=configValue) - process.stdin.write(cmd.encode() + b'\n') - process.stdin.flush() - time.sleep(1) - - process.terminate() - text = process.stdout.read().decode() - - lines = text.split('\n') - b = len(lines) - for i in range(b): - line = lines[i] - if "dumping cells as values of TLB type" in line: - a = i + 2 - break - #end for - - for i in range(a, b): - line = lines[i] - if '(' in line: - start = i - break - #end for - - for i in range(a, b): - line = lines[i] - if '>' in line: - end = i - break - #end for - - buff = lines[start:end] - text = "".join(buff) - newData = self.Tlb2Json(text) - newFileName = self.tempDir + "data1diff" - file = open(newFileName, 'wt') - newText = json.dumps(newData, indent=2) - file.write(newText) - file.close() - - oldData = self.GetConfig(configId) - oldFileName = self.tempDir + "data2diff" - file = open(oldFileName, 'wt') - oldText = json.dumps(oldData, indent=2) - file.write(oldText) - file.close() - - print(oldText) - args = ["diff", "--color", oldFileName, newFileName] - subprocess.run(args) - #end define - def GetComplaints(self, electionId=None, past=False): # Get buffer bname = "complaints" + str(past) @@ -2965,19 +2840,6 @@ def GetVotedComplaints(self, complaints: dict): return result #end define - def GetDestinationAddr(self, destination): - if self.IsAddrB64(destination): - pass - elif self.IsAddrFull(destination): - destination = self.AddrFull2AddrB64(destination) - else: - walletsNameList = self.GetWalletsNameList() - if destination in walletsNameList: - wallet = self.GetLocalWallet(destination) - destination = wallet.addrB64 - return destination - #end define - def AddrFull2AddrB64(self, addrFull, bounceable=True): if addrFull is None or "None" in addrFull: return diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index ff12e510..1f2cc273 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -80,34 +80,6 @@ def inject_globals(func): console.AddItem("set", inject_globals(SetSettings), local.translate("set_cmd")) console.AddItem("rollback", inject_globals(rollback_to_mtc1), local.translate("rollback_cmd")) - console.AddItem("seqno", inject_globals(Seqno), local.translate("seqno_cmd")) - console.AddItem("getconfig", inject_globals(GetConfig), local.translate("getconfig_cmd")) - console.AddItem("get_pool_data", inject_globals(GetPoolData), local.translate("get_pool_data_cmd")) - - console.AddItem("nw", inject_globals(CreatNewWallet), local.translate("nw_cmd")) - console.AddItem("aw", inject_globals(ActivateWallet), local.translate("aw_cmd")) - console.AddItem("wl", inject_globals(PrintWalletsList), local.translate("wl_cmd")) - console.AddItem("iw", inject_globals(ImportWallet), local.translate("iw_cmd")) - console.AddItem("swv", inject_globals(SetWalletVersion), local.translate("swv_cmd")) - console.AddItem("ew", inject_globals(ExportWallet), local.translate("ex_cmd")) - console.AddItem("dw", inject_globals(DeleteWallet), local.translate("dw_cmd")) - - console.AddItem("vas", inject_globals(ViewAccountStatus), local.translate("vas_cmd")) - console.AddItem("vah", inject_globals(ViewAccountHistory), local.translate("vah_cmd")) - console.AddItem("mg", inject_globals(MoveCoins), local.translate("mg_cmd")) - console.AddItem("mgtp", inject_globals(MoveCoinsThroughProxy), local.translate("mgtp_cmd")) - - console.AddItem("nb", inject_globals(CreatNewBookmark), local.translate("nb_cmd")) - console.AddItem("bl", inject_globals(PrintBookmarksList), local.translate("bl_cmd")) - console.AddItem("db", inject_globals(DeleteBookmark), local.translate("db_cmd")) - - console.AddItem("ol", inject_globals(PrintOffersList), local.translate("ol_cmd")) - console.AddItem("od", inject_globals(OfferDiff), local.translate("od_cmd")) - - console.AddItem("el", inject_globals(PrintElectionEntriesList), local.translate("el_cmd")) - console.AddItem("vl", inject_globals(PrintValidatorList), local.translate("vl_cmd")) - console.AddItem("cl", inject_globals(PrintComplaintsList), local.translate("cl_cmd")) - #console.AddItem("xrestart", inject_globals(Xrestart), local.translate("xrestart_cmd")) #console.AddItem("xlist", inject_globals(Xlist), local.translate("xlist_cmd")) #console.AddItem("gpk", inject_globals(GetPubKey), local.translate("gpk_cmd")) @@ -118,15 +90,23 @@ def inject_globals(func): module = CustomOverlayModule(ton, local) module.add_console_commands(console) - from modules.collator_config import CollatorConfigModule - module = CollatorConfigModule(ton, local) - module.add_console_commands(console) - if ton.using_validator(): from modules.validator import ValidatorModule module = ValidatorModule(ton, local) module.add_console_commands(console) + from modules.collator_config import CollatorConfigModule + module = CollatorConfigModule(ton, local) + module.add_console_commands(console) + + from modules.wallet import WalletModule + module = WalletModule(ton, local) + module.add_console_commands(console) + + from modules.utilities import UtilitiesModule + module = UtilitiesModule(ton, local) + module.add_console_commands(console) + if ton.using_pool(): # add basic pool functions (pools_list, delete_pool, import_pool) from modules.pool import PoolModule module = PoolModule(ton, local) @@ -501,15 +481,6 @@ def CheckTonUpdate(local): color_print(local.translate("ton_update_available")) #end define -def PrintTest(local, args): - print(json.dumps(local.buffer, indent=2)) -#end define - -def sl(ton, args): - Slashing(ton.local, ton) -#end define - - def mode_status(ton, args): from modules import get_mode modes = ton.get_modes() @@ -875,408 +846,6 @@ def GetColorTime(datetime, timestamp): return result #end define -def Seqno(ton, args): - try: - walletName = args[0] - except: - color_print("{red}Bad args. Usage:{endc} seqno ") - return - wallet = ton.GetLocalWallet(walletName) - seqno = ton.GetSeqno(wallet) - print(walletName, "seqno:", seqno) -#end define - -def CreatNewWallet(ton, args): - version = "v1" - try: - if len(args) == 0: - walletName = ton.GenerateWalletName() - workchain = 0 - else: - workchain = int(args[0]) - walletName = args[1] - if len(args) > 2: - version = args[2] - if len(args) == 4: - subwallet = int(args[3]) - else: - subwallet = 698983191 + workchain # 0x29A9A317 + workchain - except: - color_print("{red}Bad args. Usage:{endc} nw [ ]") - return - wallet = ton.CreateWallet(walletName, workchain, version, subwallet=subwallet) - table = list() - table += [["Name", "Workchain", "Address"]] - table += [[wallet.name, wallet.workchain, wallet.addrB64_init]] - print_table(table) -#end define - -def ActivateWallet(local, ton, args): - try: - walletName = args[0] - except Exception as err: - walletName = "all" - if walletName == "all": - ton.WalletsCheck() - else: - wallet = ton.GetLocalWallet(walletName) - ton.ActivateWallet(wallet) - color_print("ActivateWallet - {green}OK{endc}") -#end define - -def PrintWalletsList(ton, args): - table = list() - table += [["Name", "Status", "Balance", "Ver", "Wch", "Address"]] - data = ton.GetWallets() - if (data is None or len(data) == 0): - print("No data") - return - for wallet in data: - account = ton.GetAccount(wallet.addrB64) - if account.status != "active": - wallet.addrB64 = wallet.addrB64_init - table += [[wallet.name, account.status, account.balance, wallet.version, wallet.workchain, wallet.addrB64]] - print_table(table) -#end define - -def ImportWallet(ton, args): - try: - addr = args[0] - key = args[1] - except: - color_print("{red}Bad args. Usage:{endc} iw ") - return - name = ton.ImportWallet(addr, key) - print("Wallet name:", name) -#end define - -def SetWalletVersion(ton, args): - try: - addr = args[0] - version = args[1] - except: - color_print("{red}Bad args. Usage:{endc} swv ") - return - ton.SetWalletVersion(addr, version) - color_print("SetWalletVersion - {green}OK{endc}") -#end define - -def ExportWallet(ton, args): - try: - name = args[0] - except: - color_print("{red}Bad args. Usage:{endc} ew ") - return - addr, key = ton.ExportWallet(name) - print("Wallet name:", name) - print("Address:", addr) - print("Secret key:", key) -#end define - -def DeleteWallet(ton, args): - try: - walletName = args[0] - except: - color_print("{red}Bad args. Usage:{endc} dw ") - return - if input("Are you sure you want to delete this wallet (yes/no): ") != "yes": - print("Cancel wallet deletion") - return - wallet = ton.GetLocalWallet(walletName) - wallet.Delete() - color_print("DeleteWallet - {green}OK{endc}") -#end define - -def ViewAccountStatus(ton, args): - try: - addrB64 = args[0] - except: - color_print("{red}Bad args. Usage:{endc} vas ") - return - addrB64 = ton.GetDestinationAddr(addrB64) - account = ton.GetAccount(addrB64) - version = ton.GetVersionFromCodeHash(account.codeHash) - statusTable = list() - statusTable += [["Address", "Status", "Balance", "Version"]] - statusTable += [[addrB64, account.status, account.balance, version]] - codeHashTable = list() - codeHashTable += [["Code hash"]] - codeHashTable += [[account.codeHash]] - historyTable = GetHistoryTable(ton, addrB64, 10) - print_table(statusTable) - print() - print_table(codeHashTable) - print() - print_table(historyTable) -#end define - -def ViewAccountHistory(ton, args): - try: - addr = args[0] - limit = int(args[1]) - except: - color_print("{red}Bad args. Usage:{endc} vah ") - return - table = GetHistoryTable(ton, addr, limit) - print_table(table) -#end define - -def GetHistoryTable(ton, addr, limit): - addr = ton.GetDestinationAddr(addr) - account = ton.GetAccount(addr) - history = ton.GetAccountHistory(account, limit) - table = list() - typeText = color_text("{red}{bold}{endc}") - table += [["Time", typeText, "Coins", "From/To"]] - for message in history: - if message.srcAddr is None: - continue - srcAddrFull = f"{message.srcWorkchain}:{message.srcAddr}" - destAddFull = f"{message.destWorkchain}:{message.destAddr}" - if srcAddrFull == account.addrFull: - type = color_text("{red}{bold}>>>{endc}") - fromto = destAddFull - else: - type = color_text("{blue}{bold}<<<{endc}") - fromto = srcAddrFull - fromto = ton.AddrFull2AddrB64(fromto) - #datetime = timestamp2datetime(message.time, "%Y.%m.%d %H:%M:%S") - datetime = timeago(message.time) - table += [[datetime, type, message.value, fromto]] - return table -#end define - -def MoveCoins(ton, args): - try: - walletName = args[0] - destination = args[1] - amount = args[2] - flags = args[3:] - except: - color_print("{red}Bad args. Usage:{endc} mg ") - return - wallet = ton.GetLocalWallet(walletName) - destination = ton.GetDestinationAddr(destination) - ton.MoveCoins(wallet, destination, amount, flags=flags) - color_print("MoveCoins - {green}OK{endc}") -#end define - -def MoveCoinsThroughProxy(ton, args): - try: - walletName = args[0] - destination = args[1] - amount = args[2] - except: - color_print("{red}Bad args. Usage:{endc} mgtp ") - return - wallet = ton.GetLocalWallet(walletName) - destination = ton.GetDestinationAddr(destination) - ton.MoveCoinsThroughProxy(wallet, destination, amount) - color_print("MoveCoinsThroughProxy - {green}OK{endc}") -#end define - -def CreatNewBookmark(ton, args): - try: - name = args[0] - addr = args[1] - except: - color_print("{red}Bad args. Usage:{endc} nb ") - return - if ton.IsAddr(addr): - type = "account" - #end if - - bookmark = dict() - bookmark["name"] = name - bookmark["type"] = type - bookmark["addr"] = addr - ton.AddBookmark(bookmark) - color_print("CreatNewBookmark - {green}OK{endc}") -#end define - -def PrintBookmarksList(ton, args): - data = ton.GetBookmarks() - if (data is None or len(data) == 0): - print("No data") - return - table = list() - table += [["Name", "Type", "Address", "Balance / Exp. date"]] - for item in data: - name = item.get("name") - type = item.get("type") - addr = item.get("addr") - bookmark_data = item.get("data") - table += [[name, type, addr, bookmark_data]] - print_table(table) -#end define - -def DeleteBookmark(ton, args): - try: - name = args[0] - type = args[1] - except: - color_print("{red}Bad args. Usage:{endc} db ") - return - ton.DeleteBookmark(name, type) - color_print("DeleteBookmark - {green}OK{endc}") -#end define - -def PrintOffersList(ton, args): - data = ton.GetOffers() - if (data is None or len(data) == 0): - print("No data") - return - if "--json" in args: - text = json.dumps(data, indent=2) - print(text) - else: - table = list() - table += [["Hash", "Config", "Votes", "W/L", "Approved", "Is passed"]] - for item in data: - hash = item.get("hash") - votedValidators = len(item.get("votedValidators")) - wins = item.get("wins") - losses = item.get("losses") - wl = "{0}/{1}".format(wins, losses) - approvedPercent = item.get("approvedPercent") - approvedPercent_text = "{0}%".format(approvedPercent) - isPassed = item.get("isPassed") - if "hash" not in args: - hash = Reduct(hash) - if isPassed == True: - isPassed = bcolors.green_text("true") - if isPassed == False: - isPassed = bcolors.red_text("false") - table += [[hash, item.config.id, votedValidators, wl, approvedPercent_text, isPassed]] - print_table(table) -#end define - -def OfferDiff(ton, args): - try: - offerHash = args[0] - offerHash = offerHash - except: - color_print("{red}Bad args. Usage:{endc} od ") - return - ton.GetOfferDiff(offerHash) -#end define - -def GetConfig(ton, args): - try: - configId = args[0] - configId = int(configId) - except: - color_print("{red}Bad args. Usage:{endc} gc ") - return - data = ton.GetConfig(configId) - text = json.dumps(data, indent=2) - print(text) -#end define - -def PrintComplaintsList(ton, args): - past = "past" in args - data = ton.GetComplaints(past=past) - if (data is None or len(data) == 0): - print("No data") - return - if "--json" in args: - text = json.dumps(data, indent=2) - print(text) - else: - table = list() - table += [["Election id", "ADNL", "Fine (part)", "Votes", "Approved", "Is passed"]] - for key, item in data.items(): - electionId = item.get("electionId") - adnl = item.get("adnl") - suggestedFine = item.get("suggestedFine") - suggestedFinePart = item.get("suggestedFinePart") - Fine_text = "{0} ({1})".format(suggestedFine, suggestedFinePart) - votedValidators = len(item.get("votedValidators")) - approvedPercent = item.get("approvedPercent") - approvedPercent_text = "{0}%".format(approvedPercent) - isPassed = item.get("isPassed") - if "adnl" not in args: - adnl = Reduct(adnl) - if isPassed == True: - isPassed = bcolors.green_text("true") - if isPassed == False: - isPassed = bcolors.red_text("false") - table += [[electionId, adnl, Fine_text, votedValidators, approvedPercent_text, isPassed]] - print_table(table) -#end define - -def PrintElectionEntriesList(ton, args): - past = "past" in args - data = ton.GetElectionEntries(past=past) - if (data is None or len(data) == 0): - print("No data") - return - if "--json" in args: - text = json.dumps(data, indent=2) - print(text) - else: - table = list() - table += [["ADNL", "Pubkey", "Wallet", "Stake", "Max-factor"]] - for key, item in data.items(): - adnl = item.get("adnlAddr") - pubkey = item.get("pubkey") - walletAddr = item.get("walletAddr") - stake = item.get("stake") - maxFactor = item.get("maxFactor") - if "adnl" not in args: - adnl = Reduct(adnl) - if "pubkey" not in args: - pubkey = Reduct(pubkey) - if "wallet" not in args: - walletAddr = Reduct(walletAddr) - table += [[adnl, pubkey, walletAddr, stake, maxFactor]] - print_table(table) -#end define - -def PrintValidatorList(ton, args): - past = "past" in args - data = ton.GetValidatorsList(past=past) - if (data is None or len(data) == 0): - print("No data") - return - if "--json" in args: - text = json.dumps(data, indent=2) - print(text) - else: - table = list() - table += [["ADNL", "Pubkey", "Wallet", "Efficiency", "Online"]] - for item in data: - adnl = item.get("adnlAddr") - pubkey = item.get("pubkey") - walletAddr = item.get("walletAddr") - efficiency = item.get("efficiency") - online = item.get("online") - if "adnl" not in args: - adnl = Reduct(adnl) - if "pubkey" not in args: - pubkey = Reduct(pubkey) - if "wallet" not in args: - walletAddr = Reduct(walletAddr) - if "offline" in args and online != False: - continue - if online == True: - online = bcolors.green_text("true") - if online == False: - online = bcolors.red_text("false") - table += [[adnl, pubkey, walletAddr, efficiency, online]] - print_table(table) -#end define - -def Reduct(item): - item = str(item) - if item is None: - result = None - else: - end = len(item) - result = item[0:6] + "..." + item[end-6:end] - return result -#end define - def GetSettings(ton, args): try: name = args[0] @@ -1378,21 +947,6 @@ def ImportShardOverlayCert(ton, args): ton.ImportShardOverlayCert() #end define -def GetPoolData(ton, args): - try: - pool_name = args[0] - except: - color_print("{red}Bad args. Usage:{endc} get_pool_data ") - return - if ton.IsAddr(pool_name): - pool_addr = pool_name - else: - pool = ton.GetLocalPool(pool_name) - pool_addr = pool.addrB64 - pool_data = ton.GetPoolData(pool_addr) - print(json.dumps(pool_data, indent=4)) -#end define - ### Start of the program def mytonctrl(): From 3145718a44821825e01d274051e51eebb6586f18 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 15 Aug 2024 14:19:10 +0900 Subject: [PATCH 018/206] do not print some status info if not validator mode --- mytonctrl/mytonctrl.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 1f2cc273..874f0435 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -569,9 +569,9 @@ def PrintStatus(local, ton, args): if all_status: PrintTonStatus(local, network_name, startWorkTime, totalValidators, onlineValidators, shardsNumber, offersNumber, complaintsNumber, tpsAvg) - PrintLocalStatus(local, adnl_addr, validator_index, validator_efficiency, validator_wallet, validator_account, validator_status, + PrintLocalStatus(local, ton, adnl_addr, validator_index, validator_efficiency, validator_wallet, validator_account, validator_status, db_size, db_usage, memory_info, swap_info, net_load_avg, disks_load_avg, disks_load_percent_avg, fullnode_adnl) - if all_status: + if all_status and ton.using_validator(): PrintTonConfig(local, fullConfigAddr, fullElectorAddr, config15, config17) PrintTimes(local, rootWorkchainEnabledTime_int, startWorkTime, oldStartWorkTime, config15) #end define @@ -620,7 +620,7 @@ def PrintTonStatus(local, network_name, startWorkTime, totalValidators, onlineVa print() #end define -def PrintLocalStatus(local, adnlAddr, validatorIndex, validatorEfficiency, validatorWallet, validatorAccount, validator_status, dbSize, dbUsage, memoryInfo, swapInfo, netLoadAvg, disksLoadAvg, disksLoadPercentAvg, fullnode_adnl): +def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, validatorWallet, validatorAccount, validator_status, dbSize, dbUsage, memoryInfo, swapInfo, netLoadAvg, disksLoadAvg, disksLoadPercentAvg, fullnode_adnl): if validatorWallet is None: return walletAddr = validatorWallet.addrB64 @@ -724,12 +724,14 @@ def PrintLocalStatus(local, adnlAddr, validatorIndex, validatorEfficiency, valid validatorVersion_text = local.translate("local_status_version_validator").format(validatorGitHash_text, validatorGitBranch_text) color_print(local.translate("local_status_head")) - print(validatorIndex_text) + if ton.using_validator(): + print(validatorIndex_text) print(validatorEfficiency_text) print(adnlAddr_text) print(fullnode_adnl_text) - print(walletAddr_text) - print(walletBalance_text) + if ton.using_validator(): + print(walletAddr_text) + print(walletBalance_text) print(cpuLoad_text) print(netLoad_text) print(memoryLoad_text) From 0b739cf732ae1adca9b6d7aac18729710e7d46a9 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 15 Aug 2024 14:20:56 +0900 Subject: [PATCH 019/206] rm unused --- modules/telemetry.py | 152 ------------------------------------------- 1 file changed, 152 deletions(-) delete mode 100644 modules/telemetry.py diff --git a/modules/telemetry.py b/modules/telemetry.py deleted file mode 100644 index e1441b59..00000000 --- a/modules/telemetry.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import subprocess - -import psutil - -from modules.module import MtcModule -from mytoncore.utils import parse_db_stats -from mypylib.mypylib import get_service_pid - - -class TelemetryModule(MtcModule): - - @staticmethod - def GetUname(): - data = os.uname() - result = dict( - zip('sysname nodename release version machine'.split(), data)) - result.pop("nodename") - return result - # end define - - @staticmethod - def GetMemoryInfo(): - result = dict() - data = psutil.virtual_memory() - result["total"] = round(data.total / 10 ** 9, 2) - result["usage"] = round(data.used / 10 ** 9, 2) - result["usagePercent"] = data.percent - return result - # end define - - @staticmethod - def GetSwapInfo(): - result = dict() - data = psutil.swap_memory() - result["total"] = round(data.total / 10 ** 9, 2) - result["usage"] = round(data.used / 10 ** 9, 2) - result["usagePercent"] = data.percent - return result - # end define - - @staticmethod - def GetValidatorProcessInfo(): - pid = get_service_pid("validator") - if pid == None or pid == 0: - return - p = psutil.Process(pid) - mem = p.memory_info() - result = dict() - result["cpuPercent"] = p.cpu_percent() - memory = dict() - memory["rss"] = mem.rss - memory["vms"] = mem.vms - memory["shared"] = mem.shared - memory["text"] = mem.text - memory["lib"] = mem.lib - memory["data"] = mem.data - memory["dirty"] = mem.dirty - result["memory"] = memory - # io = p.io_counters() # Permission denied: '/proc/{pid}/io' - return result - # end define - - @staticmethod - def get_db_stats(): - result = { - 'rocksdb': { - 'ok': True, - 'message': '', - 'data': {} - }, - 'celldb': { - 'ok': True, - 'message': '', - 'data': {} - }, - } - rocksdb_stats_path = '/var/ton-work/db/db_stats.txt' - celldb_stats_path = '/var/ton-work/db/celldb/db_stats.txt' - if os.path.exists(rocksdb_stats_path): - try: - result['rocksdb']['data'] = parse_db_stats(rocksdb_stats_path) - except Exception as e: - result['rocksdb']['ok'] = False - result['rocksdb']['message'] = f'failed to fetch db stats: {e}' - else: - result['rocksdb']['ok'] = False - result['rocksdb']['message'] = 'db stats file is not exists' - # end if - - if os.path.exists(celldb_stats_path): - try: - result['celldb']['data'] = parse_db_stats(celldb_stats_path) - except Exception as e: - result['celldb']['ok'] = False - result['celldb']['message'] = f'failed to fetch db stats: {e}' - else: - result['celldb']['ok'] = False - result['celldb']['message'] = 'db stats file is not exists' - # end if - return result - # end define - - @staticmethod - def get_cpu_name(): - with open('/proc/cpuinfo') as f: - for line in f: - if line.strip(): - if line.rstrip('\n').startswith('model name'): - return line.rstrip('\n').split(':')[1].strip() - return None - # end define - - @staticmethod - def is_host_virtual(): - try: - with open('/sys/class/dmi/id/product_name') as f: - product_name = f.read().strip().lower() - if 'virtual' in product_name or 'kvm' in product_name or 'qemu' in product_name or 'vmware' in product_name: - return {'virtual': True, 'product_name': product_name} - return {'virtual': False, 'product_name': product_name} - except FileNotFoundError: - return {'virtual': None, 'product_name': None} - # end define - - @staticmethod - def do_beacon_ping(host, count, timeout): - args = ['ping', '-c', str(count), '-W', str(timeout), host] - process = subprocess.run(args, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) - output = process.stdout.decode("utf-8") - avg = output.split('\n')[-2].split('=')[1].split('/')[1] - return float(avg) - # end define - - @classmethod - def get_pings_values(cls): - return { - 'beacon-eu-01.toncenter.com': cls.do_beacon_ping('beacon-eu-01.toncenter.com', 5, 10), - 'beacon-apac-01.toncenter.com': cls.do_beacon_ping('beacon-apac-01.toncenter.com', 5, 10) - } - # end define - - @staticmethod - def get_validator_disk_name(): - process = subprocess.run("df -h /var/ton-work/ | sed -n '2 p' | awk '{print $1}'", stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3, shell=True) - output = process.stdout.decode("utf-8") - return output.strip() - # end define - - def add_console_commands(self, console): ... From e7d09d6f291e7bfb9d669d6ee9832acd9e461528 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 15 Aug 2024 14:34:07 +0900 Subject: [PATCH 020/206] fixes --- mytonctrl/mytonctrl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 874f0435..a729a548 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -726,7 +726,7 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, color_print(local.translate("local_status_head")) if ton.using_validator(): print(validatorIndex_text) - print(validatorEfficiency_text) + print(validatorEfficiency_text) print(adnlAddr_text) print(fullnode_adnl_text) if ton.using_validator(): From 4825e159119c224be29b96979d347cfe345eac28 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 15 Aug 2024 14:37:20 +0900 Subject: [PATCH 021/206] rename local validator status --- mytonctrl/resources/translate.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 2caf4ec8..2a8ae49e 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -240,9 +240,9 @@ "zh_TW": "選舉狀態: {0}" }, "local_status_head": { - "en": "{cyan}===[ Local validator status ]==={endc}", - "ru": "{cyan}===[ Статус локального валидатора ]==={endc}", - "zh_TW": "{cyan}===[ 本地驗證者狀態 ]==={endc}" + "en": "{cyan}===[ Node status ]==={endc}", + "ru": "{cyan}===[ Статус ноды ]==={endc}", + "zh_TW": "{cyan}===[ 节点狀態 ]==={endc}" }, "local_status_validator_index": { "en": "Validator index: {0}", From 0832eb240dca34940240ecc8254011921450fab9 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 15 Aug 2024 14:39:32 +0900 Subject: [PATCH 022/206] remove activate_ton_storage_provider cmd --- mytonctrl/mytonctrl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index a729a548..8cbb68e8 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -129,7 +129,7 @@ def inject_globals(func): console.AddItem("cleanup", inject_globals(cleanup_validator_db), local.translate("cleanup_cmd")) console.AddItem("benchmark", inject_globals(run_benchmark), local.translate("benchmark_cmd")) - console.AddItem("activate_ton_storage_provider", inject_globals(activate_ton_storage_provider), local.translate("activate_ton_storage_provider_cmd")) + # console.AddItem("activate_ton_storage_provider", inject_globals(activate_ton_storage_provider), local.translate("activate_ton_storage_provider_cmd")) # Process input parameters opts, args = getopt.getopt(argv,"hc:w:",["config=","wallets="]) From b5b80feccd8a4d042c354b821f8740bb7ce60f98 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 15 Aug 2024 14:43:21 +0900 Subject: [PATCH 023/206] add commands description --- mytonctrl/resources/translate.json | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 2a8ae49e..3c35738f 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -439,6 +439,31 @@ "ru": "{red}Ошибка - UDP порт валидатора недоступен извне.{endc}", "zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}" }, + "add_custom_overlay_cmd": { + "en": "Add custom overlay", + "ru": "Добавить пользовательский оверлей", + "zh_TW": "添加自定義覆蓋" + }, + "list_custom_overlays_cmd": { + "en": "List participating custom overlays", + "ru": "Список участвуемых пользовательских оверлеев", + "zh_TW": "列出參與的自定義覆蓋" + }, + "delete_custom_overlay_cmd": { + "en": "Delete custom overlay", + "ru": "Удалить пользовательский оверлей", + "zh_TW": "刪除自定義覆蓋" + }, + "cleanup_cmd": { + "en": "Clean node old logs and temp files", + "ru": "Очистить старые логи и временные файлы ноды", + "zh_TW": "清理節點舊日誌和臨時文件" + }, + "benchmark_cmd": { + "en": "Run benchmark", + "ru": "Запустить бенчмарк", + "zh_TW": "運行基準測試" + }, "000a": { "en": "001", "ru": "002", From 932ada2c6f5541bd46e30785818868178d124601 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 16 Aug 2024 12:17:02 +0900 Subject: [PATCH 024/206] fix update commit --- mytonctrl/mytonctrl.py | 2 +- mytonctrl/scripts/update.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 53b8c158..da433fff 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -334,7 +334,7 @@ def check_git(input_args, default_repo, text, default_branch='master'): #end define def check_branch_exists(author, repo, branch): - if is_hex(branch): + if len(branch) >= 6 and is_hex(branch): print('Hex name detected, skip branch existence check.') return url = f"https://github.com/{author}/{repo}.git" diff --git a/mytonctrl/scripts/update.sh b/mytonctrl/scripts/update.sh index 463d422c..8bd4ecd9 100644 --- a/mytonctrl/scripts/update.sh +++ b/mytonctrl/scripts/update.sh @@ -37,8 +37,8 @@ pip3 uninstall -y mytonctrl # Update code echo "https://github.com/${author}/${repo}.git -> ${branch}" -git clone --branch ${branch} --recursive https://github.com/${author}/${repo}.git -cd ${repo} +git clone --recursive https://github.com/${author}/${repo}.git +cd ${repo} && git checkout ${branch} pip3 install -U . systemctl daemon-reload From 5e2e67294d452620175ca4ad0bbf62a4e0ba449d Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 19 Aug 2024 14:13:39 +0900 Subject: [PATCH 025/206] move mg mgtp to wallet module --- modules/utilities.py | 62 ++---------------------------------------- modules/wallet.py | 44 ++++++++++++++++++++++++++++++ mytoncore/mytoncore.py | 13 +++++++++ 3 files changed, 59 insertions(+), 60 deletions(-) diff --git a/modules/utilities.py b/modules/utilities.py index 49ff82c7..461dd076 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -19,7 +19,7 @@ def view_account_status(self, args): except: color_print("{red}Bad args. Usage:{endc} vas ") return - addrB64 = self.get_destination_addr(addrB64) + addrB64 = self.ton.get_destination_addr(addrB64) account = self.ton.GetAccount(addrB64) version = self.ton.GetVersionFromCodeHash(account.codeHash) statusTable = list() @@ -34,11 +34,10 @@ def view_account_status(self, args): print_table(codeHashTable) print() print_table(historyTable) - # end define def get_history_table(self, addr, limit): - addr = self.get_destination_addr(addr) + addr = self.ton.get_destination_addr(addr) account = self.ton.GetAccount(addr) history = self.ton.GetAccountHistory(account, limit) table = list() @@ -73,61 +72,6 @@ def view_account_history(self, args): print_table(table) # end define - def get_destination_addr(self, destination): - if self.ton.IsAddrB64(destination): - pass - elif self.ton.IsAddrFull(destination): - destination = self.ton.AddrFull2AddrB64(destination) - else: - wallets_name_list = self.ton.GetWalletsNameList() - if destination in wallets_name_list: - wallet = self.ton.GetLocalWallet(destination) - destination = wallet.addrB64 - return destination - # end define - - def move_coins(self, args): - try: - wallet_name = args[0] - destination = args[1] - amount = args[2] - flags = args[3:] - except: - color_print("{red}Bad args. Usage:{endc} mg ") - return - wallet = self.ton.GetLocalWallet(wallet_name) - destination = self.get_destination_addr(destination) - self.ton.MoveCoins(wallet, destination, amount, flags=flags) - color_print("MoveCoins - {green}OK{endc}") - # end define - - def do_move_coins_through_proxy(self, wallet, dest, coins): - self.local.add_log("start MoveCoinsThroughProxy function", "debug") - wallet1 = self.ton.CreateWallet("proxy_wallet1", 0) - wallet2 = self.ton.CreateWallet("proxy_wallet2", 0) - self.ton.MoveCoins(wallet, wallet1.addrB64_init, coins) - self.ton.ActivateWallet(wallet1) - self.ton.MoveCoins(wallet1, wallet2.addrB64_init, "alld") - self.ton.ActivateWallet(wallet2) - self.ton.MoveCoins(wallet2, dest, "alld", flags=["-n"]) - wallet1.Delete() - wallet2.Delete() - # end define - - def move_coins_through_proxy(self, args): - try: - wallet_name = args[0] - destination = args[1] - amount = args[2] - except: - color_print("{red}Bad args. Usage:{endc} mgtp ") - return - wallet = self.ton.GetLocalWallet(wallet_name) - destination = self.get_destination_addr(destination) - self.do_move_coins_through_proxy(wallet, destination, amount) - color_print("MoveCoinsThroughProxy - {green}OK{endc}") - # end define - def create_new_bookmark(self, args): try: name = args[0] @@ -396,8 +340,6 @@ def print_validator_list(self, args): def add_console_commands(self, console): console.AddItem("vas", self.view_account_status, self.local.translate("vas_cmd")) console.AddItem("vah", self.view_account_history, self.local.translate("vah_cmd")) - console.AddItem("mg", self.move_coins, self.local.translate("mg_cmd")) - console.AddItem("mgtp", self.move_coins_through_proxy, self.local.translate("mgtp_cmd")) console.AddItem("nb", self.create_new_bookmark, self.local.translate("nb_cmd")) console.AddItem("bl", self.print_bookmarks_list, self.local.translate("bl_cmd")) diff --git a/modules/wallet.py b/modules/wallet.py index d4b385c0..1e4bff9a 100644 --- a/modules/wallet.py +++ b/modules/wallet.py @@ -151,6 +151,48 @@ def delete_wallet(self, args): color_print("DeleteWallet - {green}OK{endc}") # end define + def move_coins(self, args): + try: + wallet_name = args[0] + destination = args[1] + amount = args[2] + flags = args[3:] + except: + color_print("{red}Bad args. Usage:{endc} mg ") + return + wallet = self.ton.GetLocalWallet(wallet_name) + destination = self.ton.get_destination_addr(destination) + self.ton.MoveCoins(wallet, destination, amount, flags=flags) + color_print("MoveCoins - {green}OK{endc}") + # end define + + def do_move_coins_through_proxy(self, wallet, dest, coins): + self.local.add_log("start MoveCoinsThroughProxy function", "debug") + wallet1 = self.ton.CreateWallet("proxy_wallet1", 0) + wallet2 = self.ton.CreateWallet("proxy_wallet2", 0) + self.ton.MoveCoins(wallet, wallet1.addrB64_init, coins) + self.ton.ActivateWallet(wallet1) + self.ton.MoveCoins(wallet1, wallet2.addrB64_init, "alld") + self.ton.ActivateWallet(wallet2) + self.ton.MoveCoins(wallet2, dest, "alld", flags=["-n"]) + wallet1.Delete() + wallet2.Delete() + # end define + + def move_coins_through_proxy(self, args): + try: + wallet_name = args[0] + destination = args[1] + amount = args[2] + except: + color_print("{red}Bad args. Usage:{endc} mgtp ") + return + wallet = self.ton.GetLocalWallet(wallet_name) + destination = self.ton.get_destination_addr(destination) + self.do_move_coins_through_proxy(wallet, destination, amount) + color_print("MoveCoinsThroughProxy - {green}OK{endc}") + # end define + def add_console_commands(self, console): console.AddItem("nw", self.create_new_wallet, self.local.translate("nw_cmd")) console.AddItem("aw", self.activate_wallet, self.local.translate("aw_cmd")) @@ -159,3 +201,5 @@ def add_console_commands(self, console): console.AddItem("swv", self.set_wallet_version, self.local.translate("swv_cmd")) console.AddItem("ew", self.export_wallet, self.local.translate("ex_cmd")) console.AddItem("dw", self.delete_wallet, self.local.translate("dw_cmd")) + console.AddItem("mg", self.move_coins, self.local.translate("mg_cmd")) + console.AddItem("mgtp", self.move_coins_through_proxy, self.local.translate("mgtp_cmd")) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index f1b03c8c..5d7accf2 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2840,6 +2840,19 @@ def GetVotedComplaints(self, complaints: dict): return result #end define + def get_destination_addr(self, destination): + if self.IsAddrB64(destination): + pass + elif self.IsAddrFull(destination): + destination = self.AddrFull2AddrB64(destination) + else: + wallets_name_list = self.GetWalletsNameList() + if destination in wallets_name_list: + wallet = self.GetLocalWallet(destination) + destination = wallet.addrB64 + return destination + # end define + def AddrFull2AddrB64(self, addrFull, bounceable=True): if addrFull is None or "None" in addrFull: return From 6ff9fe1af923af68dbd8bb0ece25c13db0a4314b Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 19 Aug 2024 14:15:14 +0900 Subject: [PATCH 026/206] fix ew --- modules/wallet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/wallet.py b/modules/wallet.py index 1e4bff9a..e4bc1c86 100644 --- a/modules/wallet.py +++ b/modules/wallet.py @@ -131,7 +131,7 @@ def export_wallet(self, args): except: color_print("{red}Bad args. Usage:{endc} ew ") return - addr, key = self.ton.ExportWallet(name) + addr, key = self.do_export_wallet(name) print("Wallet name:", name) print("Address:", addr) print("Secret key:", key) From 2cb73416b9123b80192c8aa1709be7000e53ec0a Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 19 Aug 2024 16:05:36 +0900 Subject: [PATCH 027/206] rm domain type from bookmarks --- modules/utilities.py | 15 ++++++-------- mytoncore/mytoncore.py | 46 ++++++++---------------------------------- 2 files changed, 14 insertions(+), 47 deletions(-) diff --git a/modules/utilities.py b/modules/utilities.py index 49ff82c7..ef418a6e 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -135,13 +135,12 @@ def create_new_bookmark(self, args): except: color_print("{red}Bad args. Usage:{endc} nb ") return - if self.ton.IsAddr(addr): - type = "account" + if not self.ton.IsAddr(addr): + raise Exception("Incorrect address") # end if bookmark = dict() bookmark["name"] = name - bookmark["type"] = type bookmark["addr"] = addr self.ton.AddBookmark(bookmark) color_print("CreatNewBookmark - {green}OK{endc}") @@ -153,24 +152,22 @@ def print_bookmarks_list(self, args): print("No data") return table = list() - table += [["Name", "Type", "Address", "Balance / Exp. date"]] + table += [["Name", "Address", "Balance / Exp. date"]] for item in data: name = item.get("name") - type = item.get("type") addr = item.get("addr") bookmark_data = item.get("data") - table += [[name, type, addr, bookmark_data]] + table += [[name, addr, bookmark_data]] print_table(table) # end define def delete_bookmark(self, args): try: name = args[0] - type = args[1] except: - color_print("{red}Bad args. Usage:{endc} db ") + color_print("{red}Bad args. Usage:{endc} db ") return - self.ton.DeleteBookmark(name, type) + self.ton.DeleteBookmark(name) color_print("DeleteBookmark - {green}OK{endc}") # end define diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index f1b03c8c..ff8d6d78 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2730,20 +2730,6 @@ def GetItemFromDict(self, data, search): return None #end define - def GetAutoTransferRules(self): - autoTransferRules = self.local.db.get("autoTransferRules") - if autoTransferRules is None: - autoTransferRules = list() - self.local.db["autoTransferRules"] = autoTransferRules - return autoTransferRules - #end define - - def AddAutoTransferRule(self, rule): - autoTransferRules = self.GetAutoTransferRules() - autoTransferRules.append(rule) - self.local.save() - #end define - def AddBookmark(self, bookmark): if "bookmarks" not in self.local.db: self.local.db["bookmarks"] = list() @@ -2760,23 +2746,11 @@ def GetBookmarks(self): return bookmarks #end define - def GetBookmarkAddr(self, type, name): - bookmarks = self.local.db.get("bookmarks", list()) - for bookmark in bookmarks: - bookmarkType = bookmark.get("type") - bookmarkName = bookmark.get("name") - bookmarkAddr = bookmark.get("addr") - if (bookmarkType == type and bookmarkName == name): - return bookmarkAddr - raise Exception("GetBookmarkAddr error: Bookmark not found") - #end define - - def DeleteBookmark(self, name, type): + def DeleteBookmark(self, name): bookmarks = self.local.db.get("bookmarks") for bookmark in bookmarks: - bookmarkType = bookmark.get("type") - bookmarkName = bookmark.get("name") - if (type == bookmarkType and name == bookmarkName): + bookmark_name = bookmark.get("name") + if name == bookmark_name: bookmarks.remove(bookmark) self.local.save() return @@ -2784,16 +2758,12 @@ def DeleteBookmark(self, name, type): #end define def WriteBookmarkData(self, bookmark): - type = bookmark.get("type") - if type == "account": - addr = bookmark.get("addr") - account = self.GetAccount(addr) - if account.status == "empty": - data = "empty" - else: - data = account.balance + addr = bookmark.get("addr") + account = self.GetAccount(addr) + if account.status == "empty": + data = "empty" else: - data = "null" + data = account.balance bookmark["data"] = data #end define From f568bbe61b2ba450bca777717038022d381c51bd Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 21 Aug 2024 13:43:19 +0900 Subject: [PATCH 028/206] fix installer set_node_argument -d --- mytonctrl/mytonctrl.py | 2 +- mytoninstaller/mytoninstaller.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 7f321214..8a90e076 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -247,7 +247,7 @@ def Installer(args): # args = ["python3", "/usr/src/mytonctrl/mytoninstaller.py"] cmd = ["python3", "-m", "mytoninstaller"] if args: - cmd += ["-c", *args] + cmd += ["-c", " ".join(args)] subprocess.run(cmd) #end define diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index 160b9210..c63111db 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -259,7 +259,7 @@ def General(local, console): Refresh(local) if "-c" in sys.argv: cx = sys.argv.index("-c") - args = sys.argv[cx+1:] + args = sys.argv[cx+1].split() Command(local, args, console) if "-e" in sys.argv: ex = sys.argv.index("-e") From 8c39ac06a3ae2bb699af428c2d5aac00c16efdc4 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 22 Aug 2024 14:19:57 +0900 Subject: [PATCH 029/206] add interactive installer --- mytoncore/functions.py | 12 ++--- mytoninstaller/settings.py | 10 +++-- scripts/install.py | 91 ++++++++++++++++++++++++++++++++++++++ scripts/install.sh | 13 +++++- 4 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 scripts/install.py diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 1acec162..36059e0a 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -55,8 +55,8 @@ def Event(local, event_name): ValidatorDownEvent(local) elif event_name == "enable_ton_storage_provider": enable_ton_storage_provider_event(local) - elif event_name == "enable_liteserver_mode": - enable_liteserver_mode(local) + elif event_name.startswith("enable_mode"): + enable_mode(local, event_name) local.exit() # end define @@ -93,10 +93,12 @@ def enable_ton_storage_provider_event(local): #end define -def enable_liteserver_mode(local): +def enable_mode(local, event_name): ton = MyTonCore(local) - ton.disable_mode('validator') - ton.enable_mode('liteserver') + mode = event_name.split("_")[-1] + if mode == "liteserver": + ton.disable_mode('validator') + ton.enable_mode(mode) #end define diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index d04c5673..45d167bb 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -34,7 +34,11 @@ def FirstNodeSettings(local): validatorAppPath = local.buffer.validator_app_path globalConfigPath = local.buffer.global_config_path vconfig_path = local.buffer.vconfig_path - archive_ttl = 2592000 if local.buffer.mode == 'liteserver' else 86400 + + if os.getenv('ARCHIVE_TTL'): + archive_ttl = int(os.getenv('ARCHIVE_TTL')) + else: + archive_ttl = 2592000 if local.buffer.mode == 'liteserver' else 86400 # Проверить конфигурацию if os.path.isfile(vconfig_path): @@ -892,8 +896,8 @@ def CreateSymlinks(local): def EnableMode(local): args = ["python3", "-m", "mytoncore", "-e"] - if local.buffer.mode == 'liteserver': - args.append("enable_liteserver_mode") + if local.buffer.mode: + args.append("enable_mode_" + local.buffer.mode) else: return args = ["su", "-l", local.buffer.user, "-c", ' '.join(args)] diff --git a/scripts/install.py b/scripts/install.py new file mode 100644 index 00000000..ab617f07 --- /dev/null +++ b/scripts/install.py @@ -0,0 +1,91 @@ +import os +import subprocess +import inquirer + + +def run_cli(): + questions = [ + inquirer.List( + "mode", + message="Select installation mode (More on https://docs.ton.org/participate/nodes/node-types)", + choices=["validator", "liteserver"], + ), + inquirer.List( + "network", + message="Select network", + choices=["Mainnet", "Testnet", "Other"], + ), + inquirer.Text( + "config", + message="Provide network config uri", + ignore=lambda x: x["network"] != "Other", # do not ask this question if network is not 'Other' + validate=lambda _, x: x.startswith("http"), + ), + inquirer.Text( + "archive-ttl", + message="Send the number of seconds to keep the block data in the node database. Default is 2592000 (30 days)", + ignore=lambda x: x["mode"] != "liteserver", # do not ask this question if mode is not liteserver + validate=lambda _, x: not x or x.isdigit(), # must be empty string or a number + # default=2592000 + ), + inquirer.List( + "validator-mode", + message="Select mode for validator usage. You can skip and set up this later", + ignore=lambda x: x["mode"] != "validator", # do not ask this question if mode is not validator + choices=["Validator wallet", "Nominator pool", "Single pool", "Liquid Staking", "Skip"], + ), + inquirer.Confirm( + "dump", + message="Do you want to download blockchain's dump? " + "This reduces synchronization time but requires to download a large file", + ), + inquirer.Confirm( + "telemetry", + message="Are you agree with sending your node performance statistics?" + ) + ] + + answers = inquirer.prompt(questions) + + return answers + + +def parse_args(answers: dict): + mode = answers["mode"] + network = answers["network"].lower() + config = answers["config"] + archive_ttl = answers["archive-ttl"] + validator_mode = answers["validator-mode"] + dump = answers["dump"] + telemetry = answers["telemetry"] + + res = f' -n {network}' + + if network not in ('mainnet', 'testnet'): + res += f' -c {config}' + + if archive_ttl: + os.putenv('ARCHIVE_TTL', archive_ttl) # set env variable + + if validator_mode: + res += f' -m {validator_mode}' + else: + res += f'-m {mode}' + + if dump: + res += ' -d' + if not telemetry: + res += ' -t' + + return res + + +def main(): + answers = run_cli() + command = parse_args(answers) + # subprocess.run('bash scripts/install.sh ' + command, shell=True) + subprocess.run('python3 install.sh ' + command, shell=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/install.sh b/scripts/install.sh index 5975351b..b7bef18b 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -67,6 +67,17 @@ while getopts ":c:tida:r:b:m:n:v:h" flag; do esac done +if (( $# == 0 )); then # no arguments + echo "Running cli installer" + wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/install.py + pip3 install inquirer + python3 install.py +# python3 scripts/install.py + exit +fi + +exit + # Set config based on network argument if [ "${network}" = "testnet" ]; then config="https://ton-blockchain.github.io/testnet-global.config.json" @@ -81,7 +92,7 @@ cpus=$(lscpu | grep "CPU(s)" | head -n 1 | awk '{print $2}') memory=$(cat /proc/meminfo | grep MemTotal | awk '{print $2}') echo "This machine has ${cpus} CPUs and ${memory}KB of Memory" -if [ "$ignore" = false ] && ([ "${cpus}" -lt "${cpu_required}" ] || [ "${memory}" -lt "${mem_required}"]); then +if [ "$ignore" = false ] && ([ "${cpus}" -lt "${cpu_required}" ] || [ "${memory}" -lt "${mem_required}" ]); then echo "Insufficient resources. Requires a minimum of "${cpu_required}" processors and "${mem_required}" RAM." exit 1 fi From 90d4e2e79c9dbc6e923ce68b2c984f991a411f47 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 22 Aug 2024 14:27:28 +0900 Subject: [PATCH 030/206] fix cli mode --- scripts/install.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/install.py b/scripts/install.py index ab617f07..367eff83 100644 --- a/scripts/install.py +++ b/scripts/install.py @@ -67,7 +67,13 @@ def parse_args(answers: dict): if archive_ttl: os.putenv('ARCHIVE_TTL', archive_ttl) # set env variable - if validator_mode: + if validator_mode and validator_mode not in ('Skip', 'Validator wallet'): + if validator_mode == 'Nominator pool': + validator_mode = 'nominator-pool' + elif validator_mode == 'Single pool': + validator_mode = 'single-pool' + elif validator_mode == 'Liquid Staking': + validator_mode = 'liquid-staking' res += f' -m {validator_mode}' else: res += f'-m {mode}' From 788bbed3732068de521778845eb468af1f294cac Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 22 Aug 2024 16:00:38 +0900 Subject: [PATCH 031/206] fix running installation script --- scripts/install.py | 2 +- scripts/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 367eff83..268838b5 100644 --- a/scripts/install.py +++ b/scripts/install.py @@ -90,7 +90,7 @@ def main(): answers = run_cli() command = parse_args(answers) # subprocess.run('bash scripts/install.sh ' + command, shell=True) - subprocess.run('python3 install.sh ' + command, shell=True) + subprocess.run('bash install.sh ' + command, shell=True) if __name__ == '__main__': diff --git a/scripts/install.sh b/scripts/install.sh index b7bef18b..ff4a83f6 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -69,7 +69,7 @@ done if (( $# == 0 )); then # no arguments echo "Running cli installer" - wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/install.py + wget https://raw.githubusercontent.com/yungwine/mytonctrl/cli/scripts/install.py pip3 install inquirer python3 install.py # python3 scripts/install.py From 48f67e1bde16b1e9ff2f681c5791e9420257d2af Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 22 Aug 2024 16:08:20 +0900 Subject: [PATCH 032/206] fix running installation script --- scripts/install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/install.py b/scripts/install.py index 268838b5..9b93c210 100644 --- a/scripts/install.py +++ b/scripts/install.py @@ -76,7 +76,7 @@ def parse_args(answers: dict): validator_mode = 'liquid-staking' res += f' -m {validator_mode}' else: - res += f'-m {mode}' + res += f' -m {mode}' if dump: res += ' -d' @@ -90,6 +90,7 @@ def main(): answers = run_cli() command = parse_args(answers) # subprocess.run('bash scripts/install.sh ' + command, shell=True) + print('bash install.sh ' + command) subprocess.run('bash install.sh ' + command, shell=True) From e80d583f0e1e3f76e4e09b8dfb35d110228af7fc Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 22 Aug 2024 16:11:14 +0900 Subject: [PATCH 033/206] fixes --- scripts/install.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index f1c2c88a..1b243d6b 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -77,8 +77,6 @@ if (( $# == 0 )); then # no arguments exit fi -exit - # Set config based on network argument if [ "${network}" = "testnet" ]; then config="https://ton-blockchain.github.io/testnet-global.config.json" From f8bc00d4af79ce342f6d0bf9ff067f589b03f9f6 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 23 Aug 2024 15:47:16 +0900 Subject: [PATCH 034/206] fix installation username --- scripts/install.py | 2 +- scripts/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 9b93c210..da2a0078 100644 --- a/scripts/install.py +++ b/scripts/install.py @@ -91,7 +91,7 @@ def main(): command = parse_args(answers) # subprocess.run('bash scripts/install.sh ' + command, shell=True) print('bash install.sh ' + command) - subprocess.run('bash install.sh ' + command, shell=True) + subprocess.run(['bash', 'install.sh'] + command.split()) if __name__ == '__main__': diff --git a/scripts/install.sh b/scripts/install.sh index 1b243d6b..8e1aa603 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -138,7 +138,7 @@ echo -e "${COLOR}[4/5]${ENDC} Running mytoninstaller" parent_name=$(ps -p $PPID -o comm=) user=$(whoami) -if [ "$parent_name" = "sudo" ] || [ "$parent_name" = "su" ]; then +if [ "$parent_name" = "sudo" ] || [ "$parent_name" = "su" ] || [ "$parent_name" = "python3" ]; then user=$(logname) fi echo "User: $user" From b12669dcccc5db4f5019fa1c9ddcdca767d41f4d Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 23 Aug 2024 22:32:04 +0900 Subject: [PATCH 035/206] change branch back to master --- scripts/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.sh b/scripts/install.sh index 8e1aa603..5cbb4b1a 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -70,7 +70,7 @@ done if (( $# == 0 )); then # no arguments echo "Running cli installer" - wget https://raw.githubusercontent.com/yungwine/mytonctrl/cli/scripts/install.py + wget https://raw.githubusercontent.com/ton-blockchain/mytonctrl/master/scripts/install.py pip3 install inquirer python3 install.py # python3 scripts/install.py From f73c2af662d38ebfb160226c49fa0409d4626c58 Mon Sep 17 00:00:00 2001 From: enchanted-elephant1 Date: Fri, 23 Aug 2024 14:13:40 -0400 Subject: [PATCH 036/206] feat: Enhance dump download with aria2 for improved speed and reliability - Switched from curl to aria2c for downloading the dump using 8 parallel connections. - Updated apt install command to include aria2. - Fixed typo in log message from "fuction" to "function". --- mytoninstaller/settings.py | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index d04c5673..cc7c64dd 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -86,28 +86,28 @@ def FirstNodeSettings(local): def DownloadDump(local): - dump = local.buffer.dump - if dump == False: - return - #end if - - local.add_log("start DownloadDump fuction", "debug") - url = "https://dump.ton.org" - dumpSize = requests.get(url + "/dumps/latest.tar.size.archive.txt").text - print("dumpSize:", dumpSize) - needSpace = int(dumpSize) * 3 - diskSpace = psutil.disk_usage("/var") - if needSpace > diskSpace.free: - return - #end if - - # apt install - cmd = "apt install plzip pv curl -y" - os.system(cmd) - - # download dump - cmd = "curl -s {url}/dumps/latest.tar.lz | pv | plzip -d -n8 | tar -xC /var/ton-work/db".format(url=url) - os.system(cmd) + dump = local.buffer.dump + if dump == False: + return + #end if + + local.add_log("start DownloadDump function", "debug") + url = "https://dump.ton.org" + dumpSize = requests.get(url + "/dumps/latest.tar.size.archive.txt").text + print("dumpSize:", dumpSize) + needSpace = int(dumpSize) * 3 + diskSpace = psutil.disk_usage("/var") + if needSpace > diskSpace.free: + return + #end if + + # apt install + cmd = "apt install plzip pv aria2 curl -y" + os.system(cmd) + + # download dump using aria2c with 8 connections + cmd = "aria2c -x 8 -s 8 -c {url}/dumps/latest.tar.lz -o - | pv | plzip -d -n8 | tar -xC /var/ton-work/db".format(url=url) + os.system(cmd) #end define From 00fbb95ecbc6e8c813117320fac8533c32eb5bfa Mon Sep 17 00:00:00 2001 From: enchanted-elephant1 Date: Fri, 23 Aug 2024 14:18:35 -0400 Subject: [PATCH 037/206] tmp-file-first. --- mytoninstaller/settings.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index cc7c64dd..84260853 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -105,8 +105,13 @@ def DownloadDump(local): cmd = "apt install plzip pv aria2 curl -y" os.system(cmd) - # download dump using aria2c with 8 connections - cmd = "aria2c -x 8 -s 8 -c {url}/dumps/latest.tar.lz -o - | pv | plzip -d -n8 | tar -xC /var/ton-work/db".format(url=url) + # download dump using aria2c to a temporary file + temp_file = "/tmp/latest.tar.lz" + cmd = "aria2c -x 8 -s 8 -c {url}/dumps/latest.tar.lz -o {temp_file}".format(url=url, temp_file=temp_file) + os.system(cmd) + + # process the downloaded file + cmd = "pv {temp_file} | plzip -d -n8 | tar -xC /var/ton-work/db" os.system(cmd) #end define From 56c5236c7912663d24da28d5282ee26fcd73f50a Mon Sep 17 00:00:00 2001 From: enchanted-elephant1 Date: Fri, 23 Aug 2024 14:21:31 -0400 Subject: [PATCH 038/206] cleanup --- mytoninstaller/settings.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 84260853..96b470ba 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -84,7 +84,6 @@ def FirstNodeSettings(local): StartValidator(local) #end define - def DownloadDump(local): dump = local.buffer.dump if dump == False: @@ -113,8 +112,13 @@ def DownloadDump(local): # process the downloaded file cmd = "pv {temp_file} | plzip -d -n8 | tar -xC /var/ton-work/db" os.system(cmd) -#end define + # clean up the temporary file after processing + if os.path.exists(temp_file): + os.remove(temp_file) + local.add_log("Temporary file {temp_file} removed".format(temp_file=temp_file), "debug") + #end if +#end define def FirstMytoncoreSettings(local): local.add_log("start FirstMytoncoreSettings fuction", "debug") From 549fa79b83b677fc1b6fa7fd5c8837c94743cb33 Mon Sep 17 00:00:00 2001 From: sonofmom Date: Sun, 1 Sep 2024 00:18:17 +0200 Subject: [PATCH 039/206] Fix parsing of service file ExecStart line --- mytoninstaller/node_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytoninstaller/node_args.py b/mytoninstaller/node_args.py index ec99eea3..af7b8380 100644 --- a/mytoninstaller/node_args.py +++ b/mytoninstaller/node_args.py @@ -10,7 +10,7 @@ def get_validator_service(): def get_node_start_command(): service = get_validator_service() for line in service.split('\n'): - if 'ExecStart' in line: + if line.startswith('ExecStart'): return line.split('=')[1].strip() #end define From ec7226d3ed1b5f5f3256a0e37c175a91a9ab1b32 Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Mon, 2 Sep 2024 11:12:14 +0300 Subject: [PATCH 040/206] update wget url --- scripts/install.py | 7 ------- scripts/install.sh | 6 ++---- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index da2a0078..ed863c89 100644 --- a/scripts/install.py +++ b/scripts/install.py @@ -38,10 +38,6 @@ def run_cli(): "dump", message="Do you want to download blockchain's dump? " "This reduces synchronization time but requires to download a large file", - ), - inquirer.Confirm( - "telemetry", - message="Are you agree with sending your node performance statistics?" ) ] @@ -57,7 +53,6 @@ def parse_args(answers: dict): archive_ttl = answers["archive-ttl"] validator_mode = answers["validator-mode"] dump = answers["dump"] - telemetry = answers["telemetry"] res = f' -n {network}' @@ -80,8 +75,6 @@ def parse_args(answers: dict): if dump: res += ' -d' - if not telemetry: - res += ' -t' return res diff --git a/scripts/install.sh b/scripts/install.sh index 5cbb4b1a..fe5ac3d6 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -15,7 +15,6 @@ fi author="ton-blockchain" repo="mytonctrl" branch="master" -mode="validator" network="mainnet" ton_node_version="master" # Default version @@ -68,12 +67,11 @@ while getopts ":c:tida:r:b:m:n:v:h" flag; do done -if (( $# == 0 )); then # no arguments +if [ "${mode}" = "" ]; then # no mode echo "Running cli installer" - wget https://raw.githubusercontent.com/ton-blockchain/mytonctrl/master/scripts/install.py + wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/install.py pip3 install inquirer python3 install.py -# python3 scripts/install.py exit fi From b016cbabb84eac3cad0daf94ff6ab019df5bfb94 Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Mon, 2 Sep 2024 11:23:20 +0300 Subject: [PATCH 041/206] add console commands history --- mypyconsole | 2 +- mytonctrl/mytonctrl.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mypyconsole b/mypyconsole index a8f9f569..9b730946 160000 --- a/mypyconsole +++ b/mypyconsole @@ -1 +1 @@ -Subproject commit a8f9f56972192247f37ca8f43a5e723e29994c60 +Subproject commit 9b730946fc7998a6b294fe9adb7a77ce1b2b6eef diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 7f601d14..499036d1 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -66,6 +66,7 @@ def inject_globals(func): console.name = "MyTonCtrl" console.startFunction = inject_globals(PreUp) console.debug = ton.GetSettings("debug") + console.local = local console.AddItem("update", inject_globals(Update), local.translate("update_cmd")) console.AddItem("upgrade", inject_globals(Upgrade), local.translate("upgrade_cmd")) From d0a2de520664a094065dbbceaacbb9fe6eb092f7 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 5 Sep 2024 12:03:34 +0900 Subject: [PATCH 042/206] fix upgrade.sh --- mytonctrl/scripts/upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytonctrl/scripts/upgrade.sh b/mytonctrl/scripts/upgrade.sh index a7922260..81a9fd0b 100644 --- a/mytonctrl/scripts/upgrade.sh +++ b/mytonctrl/scripts/upgrade.sh @@ -66,7 +66,7 @@ ls -A1 | xargs rm -rf # Update code echo "https://github.com/${author}/${repo}.git -> ${branch}" git clone --recursive https://github.com/${author}/${repo}.git . -cd ${repo} && git checkout ${branch} +git checkout ${branch} export CC=/usr/bin/clang export CXX=/usr/bin/clang++ export CCACHE_DISABLE=1 From 2fe09d1298a12143f34c109258549f8716a6bf43 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 5 Sep 2024 13:12:46 +0900 Subject: [PATCH 043/206] add warning to subscribe to @tonstatus --- modules/__init__.py | 1 + mytonctrl/mytonctrl.py | 6 ++++++ mytonctrl/resources/translate.json | 8 ++++---- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/__init__.py b/modules/__init__.py index 8b92c260..d639c4fb 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -54,6 +54,7 @@ class Setting: 'useDefaultCustomOverlays': Setting(None, True, 'Participate in default custom overlays node eligible to'), 'defaultCustomOverlaysUrl': Setting(None, 'https://ton-blockchain.github.io/fallback_custom_overlays.json', 'Default custom overlays config url'), 'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'), + 'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'), } diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 9e35bf38..56761ff8 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -467,11 +467,17 @@ def check_vps(local, ton): color_print(f"Virtualization detected: {data['product_name']}") #end define +def check_tg_channel(local, ton): + if ton.using_validator() and ton.local.db.get("subscribe_tg_channel") is None: + print_warning(local, "subscribe_tg_channel_warning") +#end difine + def warnings(local, ton): check_disk_usage(local, ton) check_sync(local, ton) check_validator_balance(local, ton) check_vps(local, ton) + check_tg_channel(local, ton) #end define def CheckTonUpdate(local): diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 60f5bd18..733779cc 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -419,10 +419,10 @@ "ru": "{green}Доступно обновление TON. {red}Пожалуйста, обновите его с помощью команды `upgrade`.{endc}", "zh_TW": "{green}TON 有可用更新. {red}請使用 `upgrade` 命令進行更新.{endc}" }, - "disk_usage_warning": { - "en": "{red} Disk is almost full, clean the TON database immediately: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", - "ru": "{red} Диск почти заполнен, немедленно очистите базу данных TON: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", - "zh_TW": "{red} 磁盤幾乎滿了，立即清理 TON 數據庫: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}" + "subscribe_tg_channel_warning": { + "en": "{red} Make sure you are subscribed to the TON validators channel on Telegram: https://t.me/tonstatus {endc}\nTo disable this warning use command `set subscribe_tg_channel true`", + "ru": "{red} Убедитесь, что вы подписаны на канал валидаторов TON в Telegram: https://t.me/tonstatus {endc}\nЧтобы отключить это предупреждение, используйте команду `set subscribe_tg_channel true`", + "zh_TW": "{red} 確保您已訂閱了 Telegram 上的 TON 驗證者頻道: https://t.me/tonstatus {endc}\n要禁用此警告，請使用命令 `set subscribe_tg_channel true`" }, "sync_warning": { "en": "{red} Node is out of sync. The displayed status is incomplete. {endc}", From 84c0fd8ba95c9087504dcde1968d873d9a1b948b Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 5 Sep 2024 20:49:55 +0900 Subject: [PATCH 044/206] add slashed warning --- mytonctrl/mytonctrl.py | 22 ++++++++++++++----- mytonctrl/resources/translate.json | 35 +++++++++++++++++------------- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 56761ff8..cb3c5c24 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -472,12 +472,24 @@ def check_tg_channel(local, ton): print_warning(local, "subscribe_tg_channel_warning") #end difine +def check_slashed(local, ton): + config32 = ton.GetConfig32() + save_complaints = ton.GetSaveComplaints() + complaints = save_complaints.get(str(config32['startWorkTime'])) + if not complaints: + return + for c in complaints.values(): + if c["adnl"] == ton.GetAdnlAddr() and c["isPassed"]: + print_warning(local, "slashed_warning") +#end define + def warnings(local, ton): - check_disk_usage(local, ton) - check_sync(local, ton) - check_validator_balance(local, ton) - check_vps(local, ton) - check_tg_channel(local, ton) + local.try_function(check_disk_usage, args=[local, ton]) + local.try_function(check_sync, args=[local, ton]) + local.try_function(check_validator_balance, args=[local, ton]) + local.try_function(check_vps, args=[local, ton]) + local.try_function(check_tg_channel, args=[local, ton]) + local.try_function(check_slashed, args=[local, ton]) #end define def CheckTonUpdate(local): diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 733779cc..632158b4 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -410,9 +410,9 @@ "zh_TW": "{red}這個版本已經過時了。請更新至第二版本: `update mytonctrl2`{endc}" }, "disk_usage_warning": { - "en": "{red} Disk is almost full, clean the TON database immediately: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", - "ru": "{red} Диск почти заполнен, немедленно очистите базу данных TON: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", - "zh_TW": "{red} 磁盤幾乎滿了，立即清理 TON 數據庫: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}" + "en": "{red}Disk is almost full, clean the TON database immediately: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", + "ru": "{red}Диск почти заполнен, немедленно очистите базу данных TON: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}", + "zh_TW": "{red}磁盤幾乎滿了，立即清理 TON 數據庫: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming {endc}" }, "ton_update_available": { "en": "{green}TON update available. {red}Please update it with `upgrade` command.{endc}", @@ -420,30 +420,35 @@ "zh_TW": "{green}TON 有可用更新. {red}請使用 `upgrade` 命令進行更新.{endc}" }, "subscribe_tg_channel_warning": { - "en": "{red} Make sure you are subscribed to the TON validators channel on Telegram: https://t.me/tonstatus {endc}\nTo disable this warning use command `set subscribe_tg_channel true`", - "ru": "{red} Убедитесь, что вы подписаны на канал валидаторов TON в Telegram: https://t.me/tonstatus {endc}\nЧтобы отключить это предупреждение, используйте команду `set subscribe_tg_channel true`", - "zh_TW": "{red} 確保您已訂閱了 Telegram 上的 TON 驗證者頻道: https://t.me/tonstatus {endc}\n要禁用此警告，請使用命令 `set subscribe_tg_channel true`" + "en": "{red}Make sure you are subscribed to the TON validators channel on Telegram: https://t.me/tonstatus {endc}\nTo disable this warning use command `set subscribe_tg_channel true`", + "ru": "{red}Убедитесь, что вы подписаны на канал валидаторов TON в Telegram: https://t.me/tonstatus {endc}\nЧтобы отключить это предупреждение, используйте команду `set subscribe_tg_channel true`", + "zh_TW": "{red}確保您已訂閱了 Telegram 上的 TON 驗證者頻道: https://t.me/tonstatus {endc}\n要禁用此警告，請使用命令 `set subscribe_tg_channel true`" }, "sync_warning": { - "en": "{red} Node is out of sync. The displayed status is incomplete. {endc}", - "ru": "{red} Нода не синхронизирована с сетью. Отображаемый статус не полный. {endc}", - "zh_TW": "{red} 节点不与网络同步。顯示的狀態不完整。 {endc}" + "en": "{red}Node is out of sync. The displayed status is incomplete. {endc}", + "ru": "{red}Нода не синхронизирована с сетью. Отображаемый статус не полный. {endc}", + "zh_TW": "{red}节点不与网络同步。顯示的狀態不完整。 {endc}" }, "validator_balance_warning": { - "en": "{red} Validator wallet balance is low. {endc}", - "ru": "{red} Баланс кошелька валидатора низкий. {endc}", - "zh_TW": "{red} 驗證者錢包餘額不足。 {endc}" + "en": "{red}Validator wallet balance is low. {endc}", + "ru": "{red}Баланс кошелька валидатора низкий. {endc}", + "zh_TW": "{red}驗證者錢包餘額不足。 {endc}" }, "vps_warning": { - "en": "{red} Validator is running on a VPS. Use a dedicated server for better performance. {endc}", - "ru": "{red} Валидатор работает на VPS. Используйте выделенный сервер (дедик) для лучшей производительности. {endc}", - "zh_TW": "{red} 驗證者在 VPS 上運行。使用專用服務器以獲得更好的性能。 {endc}" + "en": "{red}Validator is running on a VPS. Use a dedicated server for better performance. {endc}", + "ru": "{red}Валидатор работает на VPS. Используйте выделенный сервер (дедик) для лучшей производительности. {endc}", + "zh_TW": "{red}驗證者在 VPS 上運行。使用專用服務器以獲得更好的性能。 {endc}" }, "vport_error": { "en": "{red}Error - UDP port of the validator is not accessible from the outside.{endc}", "ru": "{red}Ошибка - UDP порт валидатора недоступен извне.{endc}", "zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}" }, + "slashed_warning": { + "en": "{red}You were fined by 101 TON for low efficiency in the previous round.{endc}", + "ru": "{red}Вы были оштрафованы на 101 TON за низкую эффективность в предыдущем раунде.{endc}", + "zh_TW": "{red}您因上一輪效率低而被罰款 101 TON。{endc}" + }, "add_custom_overlay_cmd": { "en": "Add custom overlay", "ru": "Добавить пользовательский оверлей", From b379a026d906c3738d8e67b63f8c5135e4623088 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 6 Sep 2024 13:13:34 +0900 Subject: [PATCH 045/206] add id to vl --- modules/utilities.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/utilities.py b/modules/utilities.py index 9660a293..958a564a 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -302,7 +302,8 @@ def print_election_entries_list(self, args): def print_validator_list(self, args): past = "past" in args - data = self.ton.GetValidatorsList(past=past) + fast = "fast" in args + data = self.ton.GetValidatorsList(past=past, fast=fast) if data is None or len(data) == 0: print("No data") return @@ -311,8 +312,8 @@ def print_validator_list(self, args): print(text) else: table = list() - table += [["ADNL", "Pubkey", "Wallet", "Efficiency", "Online"]] - for item in data: + table += [["id", "ADNL", "Pubkey", "Wallet", "Efficiency", "Online"]] + for i, item in enumerate(data): adnl = item.get("adnlAddr") pubkey = item.get("pubkey") walletAddr = item.get("walletAddr") @@ -330,7 +331,7 @@ def print_validator_list(self, args): online = bcolors.green_text("true") if not online: online = bcolors.red_text("false") - table += [[adnl, pubkey, walletAddr, efficiency, online]] + table += [[str(i), adnl, pubkey, walletAddr, efficiency, online]] print_table(table) # end define From 5d6365171adf1768a425c230a77045938f729a86 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 6 Sep 2024 13:15:50 +0900 Subject: [PATCH 046/206] do not create complaints that already been voted --- mytoncore/mytoncore.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 4e2dea7e..fe2ffb37 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2491,6 +2491,8 @@ def CheckValidators(self, start, end): electionId = start complaints = self.GetComplaints(electionId) valid_complaints = self.get_valid_complaints(complaints, electionId) + voted_complaints = self.GetVotedComplaints(complaints) + voted_complaints_pseudohashes = [complaint['pseudohash'] for complaint in voted_complaints.values()] data = self.GetValidatorsLoad(start, end, saveCompFiles=True) fullElectorAddr = self.GetFullElectorAddr() wallet = self.GetValidatorWallet(mode="vote") @@ -2510,7 +2512,7 @@ def CheckValidators(self, start, end): var2 = item.get("var2") pubkey = item.get("pubkey") pseudohash = pubkey + str(electionId) - if pseudohash in valid_complaints: + if pseudohash in valid_complaints or pseudohash in voted_complaints_pseudohashes: # do not create complaints that already created or voted by ourself continue if item['id'] >= config['mainValidators']: # do not create complaints for non-masterchain validators continue From 8ac9d4ba7d2089544efd795aa59738a0813e0f3a Mon Sep 17 00:00:00 2001 From: Maksim Kurbatov <94808996+yungwine@users.noreply.github.com> Date: Tue, 10 Sep 2024 21:42:30 +0900 Subject: [PATCH 047/206] fix cli install in single nominator mode --- scripts/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.py b/scripts/install.py index ed863c89..aa43c0ac 100644 --- a/scripts/install.py +++ b/scripts/install.py @@ -66,7 +66,7 @@ def parse_args(answers: dict): if validator_mode == 'Nominator pool': validator_mode = 'nominator-pool' elif validator_mode == 'Single pool': - validator_mode = 'single-pool' + validator_mode = 'single-nominator' elif validator_mode == 'Liquid Staking': validator_mode = 'liquid-staking' res += f' -m {validator_mode}' From 02a36df163f633a0c9350bc87c13b5b8d80acf63 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 12 Sep 2024 12:17:40 +0900 Subject: [PATCH 048/206] add backups support for mtc --- mytonctrl/mytonctrl.py | 46 +++++++++++++++++++++++++++++ mytonctrl/scripts/create_backup.sh | 44 +++++++++++++++++++++++++++ mytonctrl/scripts/restore_backup.sh | 42 ++++++++++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 mytonctrl/scripts/create_backup.sh create mode 100644 mytonctrl/scripts/restore_backup.sh diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index ba606750..c6eb43c6 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -80,6 +80,7 @@ def inject_globals(func): console.AddItem("get", inject_globals(GetSettings), local.translate("get_cmd")) console.AddItem("set", inject_globals(SetSettings), local.translate("set_cmd")) console.AddItem("rollback", inject_globals(rollback_to_mtc1), local.translate("rollback_cmd")) + console.AddItem("create_backup", inject_globals(create_backup), local.translate("create_backup_cmd")) #console.AddItem("xrestart", inject_globals(Xrestart), local.translate("xrestart_cmd")) #console.AddItem("xlist", inject_globals(Xlist), local.translate("xlist_cmd")) @@ -914,6 +915,51 @@ def disable_mode(local, ton, args): local.exit() #end define + +def create_backup(local, ton, args): + if len(args) > 2: + color_print("{red}Bad args. Usage:{endc} create_backup [path_to_archive] [-y]") + return + if '-y' not in args: + res = input(f'Node and Mytoncore services will be stopped for few seconds while backup is created, Proceed [y/n]?') + if res.lower() != 'y': + print('aborted.') + return + else: + args.pop(args.index('-y')) + command_args = ["-m", ton.local.buffer.my_work_dir] + if len(args) == 1: + command_args += ["-d", args[0]] + backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/backup.sh') + if run_as_root(["bash", backup_script_path] + command_args) == 0: + color_print("create_backup - {green}OK{endc}") + else: + color_print("create_backup - {red}Error{endc}") + + +def restore_backup(local, ton, args): + if len(args) == 0 or len(args) > 2: + color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") + return + if '-y' not in args: + res = input(f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]') + if res.lower() != 'y': + print('aborted.') + return + else: + args.pop(args.index('-y')) + print('Before proceeding, mtc will create a backup of current configuration.') + create_backup(local, ton, ['-y']) + command_args = ["-m", ton.local.buffer.my_work_dir] + command_args += ["-n", args[0]] + restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh') + if run_as_root(["bash", restore_script_path]) == 0: + color_print("restore_backup - {green}OK{endc}") + local.exit() + else: + color_print("restore_backup - {red}Error{endc}") + + def Xrestart(inputArgs): if len(inputArgs) < 2: color_print("{red}Bad args. Usage:{endc} xrestart ") diff --git a/mytonctrl/scripts/create_backup.sh b/mytonctrl/scripts/create_backup.sh new file mode 100644 index 00000000..9738025f --- /dev/null +++ b/mytonctrl/scripts/create_backup.sh @@ -0,0 +1,44 @@ +dest="backup_$(hostname)_$(date +%s).tar.gz" +mtc_dir="$HOME/.local/share/mytoncore" +# Get arguments +while getopts d:m: flag +do + case "${flag}" in + d) dest=${OPTARG};; + m) mtc_dir=${OPTARG};; + *) + echo "Flag -${flag} is not recognized. Aborting" + exit 1 ;; + esac +done + +COLOR='\033[92m' +ENDC='\033[0m' + +systemctl stop validator +systemctl stop mytoncore + +echo -e "${COLOR}[1/4]${ENDC} Stopped validator and mytoncore" + + +tmp_dir="/tmp/mytoncore/backup" +rm -rf $tmp_dir +mkdir $tmp_dir + +cp /var/ton-work/db/config.json ${tmp_dir} +cp -r /var/ton-work/db/keyring ${tmp_dir} +cp -r /var/ton-work/keys ${tmp_dir} +cp -r $mtc_dir $tmp_dir + +echo -e "${COLOR}[2/4]${ENDC} Copied files to ${tmp_dir}" + + +systemctl start validator +systemctl start mytoncore + +echo -e "${COLOR}[3/4]${ENDC} Started validator and mytoncore" + +sudo tar -zcvf ${dest} -C ${tmp_dir} . + +echo -e "${COLOR}[4/4]${ENDC} Backup successfully created in ${dest}!" +echo -e "If you wish to use archive package to migrate node to different machine please make sure to stop validator and mytoncore on donor (this) host prior to migration." diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh new file mode 100644 index 00000000..79d2a4c7 --- /dev/null +++ b/mytonctrl/scripts/restore_backup.sh @@ -0,0 +1,42 @@ +name="backup.tar.gz" +mtc_dir="$HOME/.local/share/mytoncore" +# Get arguments +while getopts n:m: flag +do + case "${flag}" in + n) name=${OPTARG};; + m) mtc_dir=${OPTARG};; + *) + echo "Flag -${flag} is not recognized. Aborting" + exit 1 ;; + esac +done + + +COLOR='\033[92m' +ENDC='\033[0m' + +systemctl stop validator +systemctl stop mytoncore + +echo -e "${COLOR}[1/3]${ENDC} Stopped validator and mytoncore" + + +tmp_dir="/tmp/mytoncore/backup" +rm -rf $tmp_dir +mkdir $tmp_dir +tar -xvzf $name -C $tmp_dir + +cp -f ${tmp_dir}/config.json /var/ton-work/db/ +cp -rf ${tmp_dir}/keyring /var/ton-work/db/ +cp -rf ${tmp_dir}/keys /var/ton-work +cp -rfT ${tmp_dir}/mytoncore $mtc_dir + +echo -e "${COLOR}[2/3]${ENDC} Extracted files from archive" + +rm /var/ton-work/db/dht-* + +systemctl start validator +systemctl start mytoncore + +echo -e "${COLOR}[3/3]${ENDC} Started validator and mytoncore" From 1412440a18af1c4a607b53481a914cf3c3680643 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 12 Sep 2024 12:24:59 +0900 Subject: [PATCH 049/206] fix restore_backup --- mytonctrl/mytonctrl.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index c6eb43c6..239d8eb5 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -81,6 +81,7 @@ def inject_globals(func): console.AddItem("set", inject_globals(SetSettings), local.translate("set_cmd")) console.AddItem("rollback", inject_globals(rollback_to_mtc1), local.translate("rollback_cmd")) console.AddItem("create_backup", inject_globals(create_backup), local.translate("create_backup_cmd")) + console.AddItem("restore_backup", inject_globals(restore_backup), local.translate("restore_backup_cmd")) #console.AddItem("xrestart", inject_globals(Xrestart), local.translate("xrestart_cmd")) #console.AddItem("xlist", inject_globals(Xlist), local.translate("xlist_cmd")) @@ -921,8 +922,12 @@ def create_backup(local, ton, args): color_print("{red}Bad args. Usage:{endc} create_backup [path_to_archive] [-y]") return if '-y' not in args: - res = input(f'Node and Mytoncore services will be stopped for few seconds while backup is created, Proceed [y/n]?') - if res.lower() != 'y': + try: + res = input(f'Node and Mytoncore services will be stopped for few seconds while backup is created, Proceed [y/n]?') + if res.lower() != 'y': + print('aborted.') + return + except KeyboardInterrupt: print('aborted.') return else: @@ -935,6 +940,7 @@ def create_backup(local, ton, args): color_print("create_backup - {green}OK{endc}") else: color_print("create_backup - {red}Error{endc}") +#end define def restore_backup(local, ton, args): @@ -942,8 +948,12 @@ def restore_backup(local, ton, args): color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") return if '-y' not in args: - res = input(f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]') - if res.lower() != 'y': + try: + res = input(f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]') + if res.lower() != 'y': + print('aborted.') + return + except KeyboardInterrupt: print('aborted.') return else: @@ -958,6 +968,7 @@ def restore_backup(local, ton, args): local.exit() else: color_print("restore_backup - {red}Error{endc}") +#end define def Xrestart(inputArgs): From 0987a2169d989e21f1a806c43cca884392fac6e7 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 12 Sep 2024 12:27:05 +0900 Subject: [PATCH 050/206] fix backups --- mytonctrl/mytonctrl.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 239d8eb5..12d8f9e6 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -922,12 +922,8 @@ def create_backup(local, ton, args): color_print("{red}Bad args. Usage:{endc} create_backup [path_to_archive] [-y]") return if '-y' not in args: - try: - res = input(f'Node and Mytoncore services will be stopped for few seconds while backup is created, Proceed [y/n]?') - if res.lower() != 'y': - print('aborted.') - return - except KeyboardInterrupt: + res = input(f'Node and Mytoncore services will be stopped for few seconds while backup is created, Proceed [y/n]?') + if res.lower() != 'y': print('aborted.') return else: @@ -948,12 +944,8 @@ def restore_backup(local, ton, args): color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") return if '-y' not in args: - try: - res = input(f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]') - if res.lower() != 'y': - print('aborted.') - return - except KeyboardInterrupt: + res = input(f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]') + if res.lower() != 'y': print('aborted.') return else: From 43e5b3e70abf1fabafef847376bb47d894646ccd Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 12 Sep 2024 12:27:44 +0900 Subject: [PATCH 051/206] fix backups --- mytonctrl/mytonctrl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 12d8f9e6..1ff10e7a 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -931,7 +931,7 @@ def create_backup(local, ton, args): command_args = ["-m", ton.local.buffer.my_work_dir] if len(args) == 1: command_args += ["-d", args[0]] - backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/backup.sh') + backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh') if run_as_root(["bash", backup_script_path] + command_args) == 0: color_print("create_backup - {green}OK{endc}") else: From 2c28d4acf137b37e645bbda51812f448fbf52ded Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 12 Sep 2024 12:29:29 +0900 Subject: [PATCH 052/206] rm verbosity from tar --- mytonctrl/scripts/create_backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytonctrl/scripts/create_backup.sh b/mytonctrl/scripts/create_backup.sh index 9738025f..52ac04e4 100644 --- a/mytonctrl/scripts/create_backup.sh +++ b/mytonctrl/scripts/create_backup.sh @@ -38,7 +38,7 @@ systemctl start mytoncore echo -e "${COLOR}[3/4]${ENDC} Started validator and mytoncore" -sudo tar -zcvf ${dest} -C ${tmp_dir} . +sudo tar -zcf ${dest} -C ${tmp_dir} . echo -e "${COLOR}[4/4]${ENDC} Backup successfully created in ${dest}!" echo -e "If you wish to use archive package to migrate node to different machine please make sure to stop validator and mytoncore on donor (this) host prior to migration." From bd9567deea165b55c0960abb3177e9a1d48da398 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 12 Sep 2024 12:36:32 +0900 Subject: [PATCH 053/206] fix backups --- mytonctrl/mytonctrl.py | 2 +- mytonctrl/scripts/create_backup.sh | 2 +- mytonctrl/scripts/restore_backup.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 1ff10e7a..c3064ad9 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -955,7 +955,7 @@ def restore_backup(local, ton, args): command_args = ["-m", ton.local.buffer.my_work_dir] command_args += ["-n", args[0]] restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh') - if run_as_root(["bash", restore_script_path]) == 0: + if run_as_root(["bash", restore_script_path] + command_args) == 0: color_print("restore_backup - {green}OK{endc}") local.exit() else: diff --git a/mytonctrl/scripts/create_backup.sh b/mytonctrl/scripts/create_backup.sh index 52ac04e4..6e6e51ff 100644 --- a/mytonctrl/scripts/create_backup.sh +++ b/mytonctrl/scripts/create_backup.sh @@ -1,4 +1,4 @@ -dest="backup_$(hostname)_$(date +%s).tar.gz" +dest="mytonctrl_backup_$(hostname)_$(date +%s).tar.gz" mtc_dir="$HOME/.local/share/mytoncore" # Get arguments while getopts d:m: flag diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index 79d2a4c7..b44b68c3 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -34,7 +34,7 @@ cp -rfT ${tmp_dir}/mytoncore $mtc_dir echo -e "${COLOR}[2/3]${ENDC} Extracted files from archive" -rm /var/ton-work/db/dht-* +rm -r /var/ton-work/db/dht-* systemctl start validator systemctl start mytoncore From c5264a8bc06df2d814edf6a28f2518ac214017ac Mon Sep 17 00:00:00 2001 From: Leon <10451228+leonlarin@users.noreply.github.com> Date: Fri, 13 Sep 2024 10:27:46 +0100 Subject: [PATCH 054/206] feat: Update install.sh to allow passing in USER arg - Added an optional `-u` argument to install.sh to allow explicitly specifying the user thats passed to `mytoninstaller` and is used for version migration. - Cleaned up formatting --- scripts/install.sh | 58 +++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index fe5ac3d6..dffe1134 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -20,19 +20,20 @@ ton_node_version="master" # Default version show_help_and_exit() { - echo 'Supported arguments:' - echo ' -c PATH Provide custom config for toninstaller.sh' - echo ' -t Disable telemetry' - echo ' -i Ignore minimum requirements' - echo ' -d Use pre-packaged dump. Reduces duration of initial synchronization.' - echo ' -a Set MyTonCtrl git repo author' - echo ' -r Set MyTonCtrl git repo' - echo ' -b Set MyTonCtrl git repo branch' - echo ' -m MODE Install MyTonCtrl with specified mode (validator or liteserver)' - echo ' -n NETWORK Specify the network (mainnet or testnet)' - echo ' -v VERSION Specify the ton node version (commit, branch, or tag)' - echo ' -h Show this help' - exit + echo 'Supported arguments:' + echo ' -c PATH Provide custom config for toninstaller.sh' + echo ' -t Disable telemetry' + echo ' -i Ignore minimum requirements' + echo ' -d Use pre-packaged dump. Reduces duration of initial synchronization.' + echo ' -a Set MyTonCtrl git repo author' + echo ' -r Set MyTonCtrl git repo' + echo ' -b Set MyTonCtrl git repo branch' + echo ' -m MODE Install MyTonCtrl with specified mode (validator or liteserver)' + echo ' -n NETWORK Specify the network (mainnet or testnet)' + echo ' -v VERSION Specify the ton node version (commit, branch, or tag)' + echo ' -u USER Specify the user to be used for MyTonCtrl installation' + echo ' -h Show this help' + exit } if [[ "${1-}" =~ ^-*h(elp)?$ ]]; then @@ -47,7 +48,7 @@ dump=false cpu_required=16 mem_required=64000000 # 64GB in KB -while getopts ":c:tida:r:b:m:n:v:h" flag; do +while getopts ":c:tida:r:b:m:n:v:u:h" flag; do case "${flag}" in c) config=${OPTARG};; t) telemetry=false;; @@ -59,10 +60,11 @@ while getopts ":c:tida:r:b:m:n:v:h" flag; do m) mode=${OPTARG};; n) network=${OPTARG};; v) ton_node_version=${OPTARG};; + u) user=${OPTARG};; h) show_help_and_exit;; *) echo "Flag -${flag} is not recognized. Aborting" - exit 1 ;; + exit 1 ;; esac done @@ -90,8 +92,8 @@ memory=$(cat /proc/meminfo | grep MemTotal | awk '{print $2}') echo "This machine has ${cpus} CPUs and ${memory}KB of Memory" if [ "$ignore" = false ] && ([ "${cpus}" -lt "${cpu_required}" ] || [ "${memory}" -lt "${mem_required}" ]); then - echo "Insufficient resources. Requires a minimum of "${cpu_required}" processors and "${mem_required}" RAM." - exit 1 + echo "Insufficient resources. Requires a minimum of "${cpu_required}" processors and "${mem_required}" RAM." + exit 1 fi echo -e "${COLOR}[2/5]${ENDC} Checking for required TON components" @@ -100,9 +102,9 @@ BIN_DIR=/usr/bin # create dirs for OSX if [[ "$OSTYPE" =~ darwin.* ]]; then - SOURCES_DIR=/usr/local/src - BIN_DIR=/usr/local/bin - mkdir -p ${SOURCES_DIR} + SOURCES_DIR=/usr/local/src + BIN_DIR=/usr/local/bin + mkdir -p ${SOURCES_DIR} fi # check TON components @@ -111,9 +113,9 @@ file2=${BIN_DIR}/ton/lite-client/lite-client file3=${BIN_DIR}/ton/validator-engine-console/validator-engine-console if [ ! -f "${file1}" ] || [ ! -f "${file2}" ] || [ ! -f "${file3}" ]; then - echo "TON does not exists, building" - wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/ton_installer.sh -O /tmp/ton_installer.sh - bash /tmp/ton_installer.sh -c ${config} -v ${ton_node_version} + echo "TON does not exists, building" + wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/ton_installer.sh -O /tmp/ton_installer.sh + bash /tmp/ton_installer.sh -c ${config} -v ${ton_node_version} fi # Cloning mytonctrl @@ -134,10 +136,12 @@ pip3 install -U . # TODO: make installation from git directly echo -e "${COLOR}[4/5]${ENDC} Running mytoninstaller" # DEBUG -parent_name=$(ps -p $PPID -o comm=) -user=$(whoami) -if [ "$parent_name" = "sudo" ] || [ "$parent_name" = "su" ] || [ "$parent_name" = "python3" ]; then - user=$(logname) +if [ "${user}" = "" ]; then # no user + parent_name=$(ps -p $PPID -o comm=) + user=$(whoami) + if [ "$parent_name" = "sudo" ] || [ "$parent_name" = "su" ] || [ "$parent_name" = "python3" ]; then + user=$(logname) + fi fi echo "User: $user" python3 -m mytoninstaller -u ${user} -t ${telemetry} --dump ${dump} -m ${mode} From afabd526ea75f8b4e1fd394b8fb2608b727be89e Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 17 Sep 2024 14:25:57 +0400 Subject: [PATCH 055/206] chown keyring in restore_backup --- mytonctrl/scripts/restore_backup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index b44b68c3..60e8bf96 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -32,6 +32,8 @@ cp -rf ${tmp_dir}/keyring /var/ton-work/db/ cp -rf ${tmp_dir}/keys /var/ton-work cp -rfT ${tmp_dir}/mytoncore $mtc_dir +chown -R validator:validator /var/ton-work/db/keyring + echo -e "${COLOR}[2/3]${ENDC} Extracted files from archive" rm -r /var/ton-work/db/dht-* From 858eba27ca0020eec4d6800cd97cc13e4ddb4fc7 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 17 Sep 2024 22:16:23 +0400 Subject: [PATCH 056/206] replace ip in node config in restore_backup --- mytonctrl/mytonctrl.py | 9 ++++++--- mytonctrl/scripts/restore_backup.sh | 14 ++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index c3064ad9..1fa59ec7 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -29,7 +29,7 @@ color_text, bcolors, Dict, - MyPyClass + MyPyClass, ip2int ) from mypyconsole.mypyconsole import MyPyConsole @@ -46,6 +46,8 @@ import sys, getopt, os +from mytoninstaller.config import get_own_ip + def Init(local, ton, console, argv): # Load translate table @@ -952,8 +954,9 @@ def restore_backup(local, ton, args): args.pop(args.index('-y')) print('Before proceeding, mtc will create a backup of current configuration.') create_backup(local, ton, ['-y']) - command_args = ["-m", ton.local.buffer.my_work_dir] - command_args += ["-n", args[0]] + ip = str(ip2int(get_own_ip())) + command_args = ["-m", ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip] + restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh') if run_as_root(["bash", restore_script_path] + command_args) == 0: color_print("restore_backup - {green}OK{endc}") diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index 60e8bf96..fccac825 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -1,11 +1,13 @@ name="backup.tar.gz" mtc_dir="$HOME/.local/share/mytoncore" +ip=0 # Get arguments -while getopts n:m: flag +while getopts n:m:i: flag do case "${flag}" in n) name=${OPTARG};; m) mtc_dir=${OPTARG};; + i) ip=${OPTARG};; *) echo "Flag -${flag} is not recognized. Aborting" exit 1 ;; @@ -19,7 +21,7 @@ ENDC='\033[0m' systemctl stop validator systemctl stop mytoncore -echo -e "${COLOR}[1/3]${ENDC} Stopped validator and mytoncore" +echo -e "${COLOR}[1/4]${ENDC} Stopped validator and mytoncore" tmp_dir="/tmp/mytoncore/backup" @@ -34,11 +36,15 @@ cp -rfT ${tmp_dir}/mytoncore $mtc_dir chown -R validator:validator /var/ton-work/db/keyring -echo -e "${COLOR}[2/3]${ENDC} Extracted files from archive" +echo -e "${COLOR}[2/4]${ENDC} Extracted files from archive" rm -r /var/ton-work/db/dht-* +python3 -c "import json;path='/var/ton-work/db/config.json';f=open(path);d=json.load(f);f.close();d['addrs'][0]['ip']=int($ip);f=open(path, 'w');f.write(json.dumps(d, indent=4));f.close()" + +echo -e "${COLOR}[3/4]${ENDC} Deleted DHT files, replaced IP in node config" + systemctl start validator systemctl start mytoncore -echo -e "${COLOR}[3/3]${ENDC} Started validator and mytoncore" +echo -e "${COLOR}[4/4]${ENDC} Started validator and mytoncore" From caa91e07b8e86b97626276525a7a4106acf11adf Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 18 Sep 2024 16:57:08 +0400 Subject: [PATCH 057/206] refactor checking local adnl con --- mytoncore/mytoncore.py | 18 ------------------ mytonctrl/mytonctrl.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index fe2ffb37..9f65af15 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2600,24 +2600,6 @@ def GetDbSize(self, exceptions="log"): return result #end define - def check_adnl(self): - telemetry = self.local.db.get("sendTelemetry", False) - check_adnl = self.local.db.get("checkAdnl", telemetry) - if not check_adnl: - return - url = 'http://45.129.96.53/adnl_check' - try: - data = self.get_local_adnl_data() - response = requests.post(url, json=data, timeout=5).json() - except Exception as e: - self.local.add_log(f'Failed to check adnl connection: {type(e)}: {e}', 'error') - return False - result = response.get("ok") - if not result: - self.local.add_log(f'Failed to check adnl connection to local node: {response.get("message")}', 'error') - return result - #end define - def get_local_adnl_data(self): def int2ip(dec): diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index ba606750..b58b3185 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf_8 -*- import base64 +import random import subprocess import json import psutil @@ -10,6 +11,8 @@ from functools import partial +import requests + from mypylib.mypylib import ( int2ip, get_git_author_and_repo, @@ -484,6 +487,36 @@ def check_slashed(local, ton): print_warning(local, "slashed_warning") #end define +def check_adnl(local, ton): + telemetry = ton.local.db.get("sendTelemetry", False) + check_adnl = ton.local.db.get("checkAdnl", telemetry) + local.add_log('Checking ADNL connection to local node', 'info') + if not check_adnl: + return + hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27', '103.106.3.171'] + hosts = random.choices(hosts, k=3) + data = ton.get_local_adnl_data() + error = '' + ok = True + for host in hosts: + url = f'http://{host}/adnl_check' + try: + response = requests.post(url, json=data, timeout=5).json() + except Exception as e: + ok = False + error = f'Failed to check ADNL connection to local node: {type(e)}: {e}' + continue + result = response.get("ok") + if result: + ok = True + break + if not result: + ok = False + error = f'Failed to check ADNL connection to local node: {response.get("message")}' + if not ok: + print_warning(local, error) +#end define + def warnings(local, ton): local.try_function(check_disk_usage, args=[local, ton]) local.try_function(check_sync, args=[local, ton]) From f7a5fe72493f2a4123c05705d4fb6ab0aa045b9c Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 18 Sep 2024 17:08:22 +0400 Subject: [PATCH 058/206] fix adnl checking * add color warning * fix check adnl * fix random host choosing --- mytonctrl/mytonctrl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index b58b3185..a4202c8c 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -221,7 +221,6 @@ def PreUp(local: MyPyClass, ton: MyTonCore): CheckMytonctrlUpdate(local) check_installer_user(local) check_vport(local, ton) - ton.check_adnl() warnings(local, ton) # CheckTonUpdate() #end define @@ -494,7 +493,7 @@ def check_adnl(local, ton): if not check_adnl: return hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27', '103.106.3.171'] - hosts = random.choices(hosts, k=3) + hosts = random.sample(hosts, k=3) data = ton.get_local_adnl_data() error = '' ok = True @@ -504,7 +503,7 @@ def check_adnl(local, ton): response = requests.post(url, json=data, timeout=5).json() except Exception as e: ok = False - error = f'Failed to check ADNL connection to local node: {type(e)}: {e}' + error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}' continue result = response.get("ok") if result: @@ -512,7 +511,7 @@ def check_adnl(local, ton): break if not result: ok = False - error = f'Failed to check ADNL connection to local node: {response.get("message")}' + error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}' if not ok: print_warning(local, error) #end define @@ -520,6 +519,7 @@ def check_adnl(local, ton): def warnings(local, ton): local.try_function(check_disk_usage, args=[local, ton]) local.try_function(check_sync, args=[local, ton]) + local.try_function(check_adnl, args=[local, ton]) local.try_function(check_validator_balance, args=[local, ton]) local.try_function(check_vps, args=[local, ton]) local.try_function(check_tg_channel, args=[local, ton]) From 0f4e5caaf924b1060498f66bb5880049b7668e61 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 18 Sep 2024 17:02:43 +0400 Subject: [PATCH 059/206] show only masterchain efficiency for masterchain validator --- modules/validator.py | 8 ++++---- mytoncore/mytoncore.py | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/validator.py b/modules/validator.py index 7a6b9ad3..0315b99c 100644 --- a/modules/validator.py +++ b/modules/validator.py @@ -61,8 +61,8 @@ def check_efficiency(self, args): else: efficiency = 100 if validator.efficiency > 100 else validator.efficiency color_efficiency = GetColorInt(efficiency, 90, logic="more", ending="%") - created = validator.blocks_created - expected = validator.blocks_expected + created = validator.master_blocks_created + expected = validator.master_blocks_expected color_print(f"Previous round efficiency: {color_efficiency} {{yellow}}({created} blocks created / {round(expected, 1)} blocks expected){{endc}}") else: print("Couldn't find this validator in the previous round") @@ -81,8 +81,8 @@ def check_efficiency(self, args): else: efficiency = 100 if validator.efficiency > 100 else validator.efficiency color_efficiency = GetColorInt(efficiency, 90, logic="more", ending="%") - created = validator.blocks_created - expected = validator.blocks_expected + created = validator.master_blocks_created + expected = validator.master_blocks_expected color_print(f"Current round efficiency: {color_efficiency} {{yellow}}({created} blocks created / {round(expected, 1)} blocks expected){{endc}}") else: print("Couldn't find this validator in the current round") diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index fe2ffb37..3f29c094 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2394,7 +2394,10 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict: wr = 0 else: wr = workBlocksCreated / workBlocksExpected - r = (mr + wr) / 2 + if masterBlocksExpected > 0: # show only masterchain efficiency for masterchain validator + r = mr + else: + r = (mr + wr) / 2 efficiency = round(r * 100, 2) if efficiency > 10: online = True @@ -2467,6 +2470,8 @@ def GetValidatorsList(self, past=False, fast=False): validator["wr"] = validatorsLoad[vid]["wr"] validator["efficiency"] = validatorsLoad[vid]["efficiency"] validator["online"] = validatorsLoad[vid]["online"] + validator["master_blocks_created"] = validatorsLoad[vid]["masterBlocksCreated"] + validator["master_blocks_expected"] = validatorsLoad[vid]["masterBlocksExpected"] validator["blocks_created"] = validatorsLoad[vid]["masterBlocksCreated"] + validatorsLoad[vid]["workBlocksCreated"] validator["blocks_expected"] = validatorsLoad[vid]["masterBlocksExpected"] + validatorsLoad[vid]["workBlocksExpected"] validator["is_masterchain"] = False From 52c143a872712204365f0f61dfa6310c3c53042d Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 19 Sep 2024 13:44:04 +0400 Subject: [PATCH 060/206] fix check_ef --- modules/validator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/validator.py b/modules/validator.py index 0315b99c..8efbc904 100644 --- a/modules/validator.py +++ b/modules/validator.py @@ -63,6 +63,9 @@ def check_efficiency(self, args): color_efficiency = GetColorInt(efficiency, 90, logic="more", ending="%") created = validator.master_blocks_created expected = validator.master_blocks_expected + if created is None: # there is no updated prev round info in cache + created = validator.blocks_created + expected = validator.blocks_expected color_print(f"Previous round efficiency: {color_efficiency} {{yellow}}({created} blocks created / {round(expected, 1)} blocks expected){{endc}}") else: print("Couldn't find this validator in the previous round") From 13a6933f3cac51c946022d37cb2770b8293eb23c Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 24 Sep 2024 13:42:26 +0400 Subject: [PATCH 061/206] improve backups --- mytonctrl/scripts/create_backup.sh | 5 ++++- mytonctrl/scripts/restore_backup.sh | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mytonctrl/scripts/create_backup.sh b/mytonctrl/scripts/create_backup.sh index 6e6e51ff..9bab1355 100644 --- a/mytonctrl/scripts/create_backup.sh +++ b/mytonctrl/scripts/create_backup.sh @@ -1,5 +1,6 @@ dest="mytonctrl_backup_$(hostname)_$(date +%s).tar.gz" mtc_dir="$HOME/.local/share/mytoncore" +user=$(logname) # Get arguments while getopts d:m: flag do @@ -38,7 +39,9 @@ systemctl start mytoncore echo -e "${COLOR}[3/4]${ENDC} Started validator and mytoncore" -sudo tar -zcf ${dest} -C ${tmp_dir} . +tar -zcf $dest -C $tmp_dir . + +chown $user:$user $dest echo -e "${COLOR}[4/4]${ENDC} Backup successfully created in ${dest}!" echo -e "If you wish to use archive package to migrate node to different machine please make sure to stop validator and mytoncore on donor (this) host prior to migration." diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index fccac825..361c7ff9 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -29,6 +29,7 @@ rm -rf $tmp_dir mkdir $tmp_dir tar -xvzf $name -C $tmp_dir +rm -rf /var/ton-work/db/keyring cp -f ${tmp_dir}/config.json /var/ton-work/db/ cp -rf ${tmp_dir}/keyring /var/ton-work/db/ cp -rf ${tmp_dir}/keys /var/ton-work From 96d3d234c0bce6243cda06dfbe97c84309b8195b Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 24 Sep 2024 14:15:14 +0400 Subject: [PATCH 062/206] fix enabling THA --- mytoncore/mytoncore.py | 5 ++- mytoninstaller/mytoninstaller.py | 4 +- .../scripts}/ton_http_api_installer.sh | 6 ++- mytoninstaller/scripts/tonhttpapiinstaller.sh | 38 ------------------- mytoninstaller/settings.py | 10 ++--- 5 files changed, 15 insertions(+), 48 deletions(-) rename {scripts => mytoninstaller/scripts}/ton_http_api_installer.sh (88%) delete mode 100755 mytoninstaller/scripts/tonhttpapiinstaller.sh diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index fe2ffb37..8d29160c 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3052,6 +3052,9 @@ def check_enable_mode(self, name): if self.using_liteserver(): raise Exception(f'Cannot enable validator mode while liteserver mode is enabled. ' f'Use `disable_mode liteserver` first.') + if name == 'liquid-staking': + from mytoninstaller.settings import enable_ton_http_api + enable_ton_http_api(self.local) def enable_mode(self, name): if name not in MODES: @@ -3604,7 +3607,7 @@ def CalculateLoanAmount(self, min_loan, max_loan, max_interest): print(f"CalculateLoanAmount data: {data}") url = "http://127.0.0.1:8801/runGetMethod" - res = requests.post(url, json=data) + res = requests.post(url, json=data, timeout=3) res_data = res.json() if res_data.get("ok") is False: error = res_data.get("error") diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index c63111db..c5e60012 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -24,7 +24,7 @@ EnableLiteServer, EnableDhtServer, EnableJsonRpc, - EnableTonHttpApi, + enable_ton_http_api, DangerousRecoveryValidatorConfigFile, CreateSymlinks, enable_ls_proxy, @@ -223,7 +223,7 @@ def Event(local, name): if name == "enableJR": EnableJsonRpc(local) if name == "enableTHA": - EnableTonHttpApi(local) + enable_ton_http_api(local) if name == "enableLSP": enable_ls_proxy(local) if name == "enableTSP": diff --git a/scripts/ton_http_api_installer.sh b/mytoninstaller/scripts/ton_http_api_installer.sh similarity index 88% rename from scripts/ton_http_api_installer.sh rename to mytoninstaller/scripts/ton_http_api_installer.sh index 9327f870..039c3bd2 100644 --- a/scripts/ton_http_api_installer.sh +++ b/mytoninstaller/scripts/ton_http_api_installer.sh @@ -30,11 +30,15 @@ chown -R ${user}:${user} ${venv_path} echo -e "${COLOR}[3/4]${ENDC} Add to startup" venv_ton_http_api="${venv_path}/bin/ton-http-api" tonlib_path="/usr/bin/ton/tonlib/libtonlibjson.so" -ls_config="/usr/bin/ton/localhost.config.json" +ls_config="/usr/bin/ton/local.config.json" cmd="from sys import path; path.append('/usr/src/mytonctrl/'); from mypylib.mypylib import add2systemd; add2systemd(name='ton_http_api', user='${user}', start='${venv_ton_http_api} --host 127.0.0.1 --port 8801 --liteserver-config ${ls_config} --cdll-path ${tonlib_path} --tonlib-keystore /tmp/tonlib_keystore/')" python3 -c "${cmd}" +systemctl daemon-reload systemctl restart ton_http_api +echo -e "Requesting masterchain info from local ton http api" +curl http://127.0.0.1:8801/getMasterchainInfo + # Конец echo -e "${COLOR}[4/4]${ENDC} ton_http_api service installation complete" exit 0 diff --git a/mytoninstaller/scripts/tonhttpapiinstaller.sh b/mytoninstaller/scripts/tonhttpapiinstaller.sh deleted file mode 100755 index c22b98e0..00000000 --- a/mytoninstaller/scripts/tonhttpapiinstaller.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -set -e - -# Проверить sudo -if [ "$(id -u)" != "0" ]; then - echo "Please run script as root" - exit 1 -fi - -# Get arguments -while getopts u: flag -do - case "${flag}" in - u) user=${OPTARG};; - esac -done - -# Цвета -COLOR='\033[92m' -ENDC='\033[0m' - -# Установка компонентов python3 -echo -e "${COLOR}[1/3]${ENDC} Installing required packages" -pip3 install -U ton-http-api - -# Установка модуля -echo -e "${COLOR}[2/3]${ENDC} Add to startup" -mkdir -p /var/ton-http-api/ton_keystore/ -chown -R $user /var/ton-http-api/ - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cmd="ton-http-api --port=8000 --logs-level=INFO --cdll-path=/usr/bin/ton/tonlib/libtonlibjson.so --liteserver-config /usr/bin/ton/local.config.json --tonlib-keystore=/var/ton-http-api/ton_keystore/ --parallel-requests-per-liteserver=1024" -${SCRIPT_DIR}/add2systemd.sh -n ton-http-api -s "${cmd}" -u ${user} -g ${user} -systemctl restart ton-http-api - -# Конец -echo -e "${COLOR}[3/3]${ENDC} TonHttpApi installation complete" -exit 0 diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 45d167bb..b2f4ad1e 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -451,12 +451,10 @@ def EnableJsonRpc(local): color_print(text) #end define -def EnableTonHttpApi(local): - local.add_log("start EnablePytonv3 function", "debug") - user = local.buffer.user - - ton_http_api_installer_path = pkg_resources.resource_filename('mytoninstaller.scripts', 'tonhttpapiinstaller.sh') - exit_code = run_as_root(["bash", ton_http_api_installer_path, "-u", user]) +def enable_ton_http_api(local): + local.add_log("start EnableTonHttpApi function", "debug") + ton_http_api_installer_path = pkg_resources.resource_filename('mytoninstaller.scripts', 'ton_http_api_installer.sh') + exit_code = run_as_root(["bash", ton_http_api_installer_path]) if exit_code == 0: text = "EnableTonHttpApi - {green}OK{endc}" else: From c5e002683ac0c96c85ffb20d28ff30928fd474f0 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 24 Sep 2024 14:36:51 +0400 Subject: [PATCH 063/206] update THA installer script --- mytoninstaller/scripts/ton_http_api_installer.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mytoninstaller/scripts/ton_http_api_installer.sh b/mytoninstaller/scripts/ton_http_api_installer.sh index 039c3bd2..fa87c42c 100644 --- a/mytoninstaller/scripts/ton_http_api_installer.sh +++ b/mytoninstaller/scripts/ton_http_api_installer.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -# Проверить sudo +# check sudo if [ "$(id -u)" != "0" ]; then echo "Please run script as root" exit 1 @@ -11,34 +11,36 @@ fi COLOR='\033[92m' ENDC='\033[0m' -# Установка компонентов python3 +# install python3 packages pip3 install virtualenv -# Подготовить папку с виртуальным окружением +# prepare the virtual environment echo -e "${COLOR}[1/4]${ENDC} Preparing the virtual environment" venv_path="/opt/virtualenv/ton_http_api" virtualenv ${venv_path} -# Установка компонентов python3 +# install python3 packages echo -e "${COLOR}[2/4]${ENDC} Installing required packages" user=$(logname) venv_pip3="${venv_path}/bin/pip3" ${venv_pip3} install ton-http-api chown -R ${user}:${user} ${venv_path} -# Прописать автозагрузку +# add to startup echo -e "${COLOR}[3/4]${ENDC} Add to startup" venv_ton_http_api="${venv_path}/bin/ton-http-api" tonlib_path="/usr/bin/ton/tonlib/libtonlibjson.so" ls_config="/usr/bin/ton/local.config.json" -cmd="from sys import path; path.append('/usr/src/mytonctrl/'); from mypylib.mypylib import add2systemd; add2systemd(name='ton_http_api', user='${user}', start='${venv_ton_http_api} --host 127.0.0.1 --port 8801 --liteserver-config ${ls_config} --cdll-path ${tonlib_path} --tonlib-keystore /tmp/tonlib_keystore/')" +cmd="from sys import path; path.append('/usr/src/mytonctrl/'); from mypylib.mypylib import add2systemd; add2systemd(name='ton_http_api', user='${user}', start='${venv_ton_http_api} --logs-level=INFO --host 127.0.0.1 --port 8801 --liteserver-config ${ls_config} --cdll-path ${tonlib_path} --tonlib-keystore /tmp/tonlib_keystore/')" python3 -c "${cmd}" systemctl daemon-reload systemctl restart ton_http_api +# check connection echo -e "Requesting masterchain info from local ton http api" +sleep 5 curl http://127.0.0.1:8801/getMasterchainInfo -# Конец +# end echo -e "${COLOR}[4/4]${ENDC} ton_http_api service installation complete" exit 0 From 8146d4e76062b0c432053f35d6a07fb051e2c4ea Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 25 Sep 2024 22:29:51 +0400 Subject: [PATCH 064/206] fix buffering funciton for GetValidatorsLoad --- mytoncore/mytoncore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 9f65af15..5c2679a6 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2350,7 +2350,7 @@ def GetOnlineValidators(self): def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict: # Get buffer - bname = f"validatorsLoad{start}{end}" + bname = f"validatorsLoad{start}{end}{saveCompFiles}" buff = self.GetFunctionBuffer(bname, timeout=60) if buff: return buff From 3b25c96b27ea784bbee4a61c287d0e70620d48b5 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 26 Sep 2024 17:26:48 +0400 Subject: [PATCH 065/206] fix DownloadDump --- mytoninstaller/settings.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index f9ef4aaa..9d8db7be 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -9,10 +9,10 @@ import pkg_resources from mypylib.mypylib import ( - add2systemd, - get_dir_from_path, - run_as_root, - color_print, + add2systemd, + get_dir_from_path, + run_as_root, + color_print, ip2int, Dict ) @@ -110,17 +110,17 @@ def DownloadDump(local): # download dump using aria2c to a temporary file temp_file = "/tmp/latest.tar.lz" - cmd = "aria2c -x 8 -s 8 -c {url}/dumps/latest.tar.lz -o {temp_file}".format(url=url, temp_file=temp_file) + cmd = f"aria2c -x 8 -s 8 -c {url}/dumps/latest.tar.lz -d / -o {temp_file}" os.system(cmd) # process the downloaded file - cmd = "pv {temp_file} | plzip -d -n8 | tar -xC /var/ton-work/db" + cmd = f"pv {temp_file} | plzip -d -n8 | tar -xC /var/ton-work/db" os.system(cmd) # clean up the temporary file after processing if os.path.exists(temp_file): os.remove(temp_file) - local.add_log("Temporary file {temp_file} removed".format(temp_file=temp_file), "debug") + local.add_log(f"Temporary file {temp_file} removed", "debug") #end if #end define From 75fdd96ff52617fb4c0dfb737c17998749c11649 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 1 Oct 2024 12:10:11 +0400 Subject: [PATCH 066/206] improve WaitTransaction seqno fetching --- mytoncore/mytoncore.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 9f65af15..e19ee8f6 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1226,8 +1226,13 @@ def WaitTransaction(self, wallet, timeout=30): steps = timeout // timesleep for i in range(steps): time.sleep(timesleep) - seqno = self.GetSeqno(wallet) + try: + seqno = self.GetSeqno(wallet) + except: + self.local.add_log("WaitTransaction error: Can't get seqno", "warning") + continue if seqno != wallet.oldseqno: + self.local.add_log("WaitTransaction success", "info") return raise Exception("WaitTransaction error: time out") #end define From c89c21a6640e77011bc486795088a82df3ac5384 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 1 Oct 2024 12:53:36 +0400 Subject: [PATCH 067/206] do not repeat WithdrawFromPool on errors --- mytoncore/mytoncore.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e19ee8f6..10754fe0 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3275,9 +3275,8 @@ def PendWithdrawFromPool(self, poolAddr, amount): #end define def HandlePendingWithdraw(self, pendingWithdraws, poolAddr): - amount = pendingWithdraws.get(poolAddr) + amount = pendingWithdraws.pop(poolAddr) self.WithdrawFromPoolProcess(poolAddr, amount) - pendingWithdraws.pop(poolAddr) #end define def GetPendingWithdraws(self): From a7d37b50919fdf1e724b4a01367618f31f5d684b Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 2 Oct 2024 13:11:06 +0400 Subject: [PATCH 068/206] allow fines 1% of stake for not working validators --- mytoncore/mytoncore.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 9f65af15..838faa9e 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2305,6 +2305,7 @@ def get_valid_complaints(self, complaints: dict, election_id: int): continue exists = False + vload = None for item in validators_load.values(): if 'fileName' not in item: continue @@ -2314,14 +2315,14 @@ def get_valid_complaints(self, complaints: dict, election_id: int): pseudohash = pubkey + str(election_id) if pseudohash == complaint['pseudohash']: exists = True - vid = item['id'] + vload = item break if not exists: self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint info was not found, probably it's wrong", "info") continue - if vid >= config32['mainValidators']: + if vload["id"] >= config32['mainValidators']: self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint created for non masterchain validator", "info") continue @@ -2330,8 +2331,13 @@ def get_valid_complaints(self, complaints: dict, election_id: int): self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine value is {complaint['suggestedFine']} ton", "info") continue if complaint['suggestedFinePart'] != 0: # https://github.com/ton-blockchain/ton/blob/5847897b3758bc9ea85af38e7be8fc867e4c133a/lite-client/lite-client.cpp#L3709 - self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine part value is {complaint['suggestedFinePart']} ton", "info") - continue + if vload["id"] < config32['mainValidators'] and vload["masterBlocksCreated"] + vload["workBlocksCreated"] == 0: # masterchain validator that created 0 blocks + if complaint['suggestedFinePart'] != 42949672: # (1LL << 32) / 100 + self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine part value is {complaint['suggestedFinePart']} ton", "info") + continue + else: + self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine part value is {complaint['suggestedFinePart']} ton", "info") + continue result[complaint['pseudohash']] = complaint return result From 6ef922be48f5dc5c45511cb21866439eccf43f09 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 2 Oct 2024 13:45:54 +0400 Subject: [PATCH 069/206] update fine warnings --- modules/validator.py | 4 ++-- mytoncore/mytoncore.py | 1 + mytonctrl/mytonctrl.py | 15 +++++++++++++-- mytonctrl/resources/translate.json | 6 +++--- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/modules/validator.py b/modules/validator.py index 7a6b9ad3..05f6e98a 100644 --- a/modules/validator.py +++ b/modules/validator.py @@ -55,7 +55,7 @@ def check_efficiency(self, args): color_print(f"Previous round time: {{yellow}}from {start_time} to {end_time}{{endc}}") if validator: if validator.is_masterchain == False: - print("Validator index is greater than 100 in the previous round - no efficiency data.") + print(f"Validator index is greater than {config32['mainValidators']} in the previous round - no efficiency data.") elif validator.get('efficiency') is None: print('Failed to get efficiency for the previous round') else: @@ -72,7 +72,7 @@ def check_efficiency(self, args): color_print(f"Current round time: {{green}}from {start_time} to {end_time}{{endc}}") if validator: if validator.is_masterchain == False: - print("Validator index is greater than 100 in the current round - no efficiency data.") + print(f"Validator index is greater than {config34['mainValidators']} in the current round - no efficiency data.") elif (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8: print("The validation round has started recently, there is not enough data yet. " "The efficiency evaluation will become more accurate towards the end of the round.") diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 838faa9e..60280f4c 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2482,6 +2482,7 @@ def GetValidatorsList(self, past=False, fast=False): validator["efficiency"] = round(validator["wr"] * 100, 2) if saveElectionEntries and adnlAddr in saveElectionEntries: validator["walletAddr"] = saveElectionEntries[adnlAddr]["walletAddr"] + validator["stake"] = saveElectionEntries[adnlAddr].get("stake") #end for # Set buffer diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 1020c5c9..1bc78e4d 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -483,11 +483,22 @@ def check_slashed(local, ton): config32 = ton.GetConfig32() save_complaints = ton.GetSaveComplaints() complaints = save_complaints.get(str(config32['startWorkTime'])) + from modules.validator import ValidatorModule + module = ValidatorModule(ton, local) + vl = ton.GetValidatorsList(past=True) + me = module.find_myself(vl) + if not me: # we were not a validator in the previous round + return if not complaints: return for c in complaints.values(): - if c["adnl"] == ton.GetAdnlAddr() and c["isPassed"]: - print_warning(local, "slashed_warning") + if c["adnl"] == me["adnlAddr"] and c["isPassed"]: + if me.get("stake"): + fine = f"""{round(c['suggestedFine'] + me["stake"] * (c['suggestedFinePart'] / (1<<32)))} TON""" + else: # unknown stake amount so just print percents + fine = f"""{round(c['suggestedFine'])} TON + {(c['suggestedFinePart'] / (1<<32)) * 100} % of stake""" + warning = local.translate("slashed_warning").format(fine) + print_warning(local, warning) #end define def check_adnl(local, ton): diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 632158b4..19dfbeec 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -445,9 +445,9 @@ "zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}" }, "slashed_warning": { - "en": "{red}You were fined by 101 TON for low efficiency in the previous round.{endc}", - "ru": "{red}Вы были оштрафованы на 101 TON за низкую эффективность в предыдущем раунде.{endc}", - "zh_TW": "{red}您因上一輪效率低而被罰款 101 TON。{endc}" + "en": "{red}You were fined by {0} for low efficiency in the previous round.{endc}", + "ru": "{red}Вы были оштрафованы на {0} за низкую эффективность в предыдущем раунде.{endc}", + "zh_TW": "{red}您因上一輪效率低而被罰款 {0}。{endc}" }, "add_custom_overlay_cmd": { "en": "Add custom overlay", From da653d8a7cd56108e61d6ebf87adca3c42949d61 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:37:44 +0400 Subject: [PATCH 070/206] add AlertBot --- modules/__init__.py | 6 +- modules/alert_bot.py | 151 +++++++++++++++++++++++++++++++++++++++++ mytoncore/functions.py | 4 ++ mytoncore/mytoncore.py | 3 + mytoncore/utils.py | 4 ++ 5 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 modules/alert_bot.py diff --git a/modules/__init__.py b/modules/__init__.py index d639c4fb..afb61e14 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -8,6 +8,7 @@ from modules.validator import ValidatorModule from modules.controller import ControllerModule from modules.liteserver import LiteserverModule +from modules.alert_bot import AlertBotModule MODES = { @@ -15,7 +16,8 @@ 'nominator-pool': NominatorPoolModule, 'single-nominator': SingleNominatorModule, 'liquid-staking': ControllerModule, - 'liteserver': LiteserverModule + 'liteserver': LiteserverModule, + 'alert-bot': AlertBotModule } @@ -55,6 +57,8 @@ class Setting: 'defaultCustomOverlaysUrl': Setting(None, 'https://ton-blockchain.github.io/fallback_custom_overlays.json', 'Default custom overlays config url'), 'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'), 'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'), + 'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'), + 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id') } diff --git a/modules/alert_bot.py b/modules/alert_bot.py new file mode 100644 index 00000000..6118ffb1 --- /dev/null +++ b/modules/alert_bot.py @@ -0,0 +1,151 @@ +import dataclasses +import time + +from modules.module import MtcModule +from mytoncore import get_hostname +from mytonctrl.utils import timestamp2utcdatetime + + +class Alert(dataclasses.dataclass): + severity: str + text: str + timeout: int + + +HOUR = 3600 + + +ALERTS = { + "low_wallet_balance": Alert( + "medium", + "Validator wallet {wallet} balance is low: {balance} TON.", + 18*HOUR + ), + "db_usage_80": Alert( + "high", + """TON DB usage > 80%. Clean the TON database: + https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming + or (and) set node\'s archive ttl to lower value.""", + 24*HOUR + ), + "db_usage_95": Alert( + "critical", + """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: + https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming + or (and) set node\'s archive ttl to lower value.""", + 6*HOUR + ), + "low_efficiency": Alert( + "high", + """Validator efficiency is low: {efficiency}%.""", + 12*HOUR + ), + "zero_block_created": Alert( + "critical", + "No blocks created for the last 6 hours.", + 6*HOUR + ), + "out_of_sync": Alert( + "critical", + "Node is out of sync on {sync} sec.", + 0 + ) +} + + +class AlertBotModule(MtcModule): + + description = 'Telegram bot alerts' + default_value = False + + def __init__(self, ton, local, *args, **kwargs): + super().__init__(ton, local, *args, **kwargs) + self.inited = False + self.hostname = None + self.bot = None + self.token = self.ton.local.db.get("BotToken") + self.chat_id = self.ton.local.db.get("ChatId") + + def send_message(self, text: str): + if self.bot is not None: + self.bot.send_message(self.chat_id, text) + else: + raise Exception("send_message error: bot is not initialized") + + def send_alert(self, alert_name: str, *args, **kwargs): + last_sent = self.get_alert_sent(alert_name) + time_ = timestamp2utcdatetime(int(time.time())) + alert = ALERTS.get(alert_name) + if alert is None: + raise Exception(f"Alert {alert_name} not found") + text = f''' +MyTonCtrl Alert {alert_name} + +Hostname: {self.hostname} +Time: {time_} ({int(time.time())}) +Severity: {alert.severity} +Next alert of this type in: {alert.timeout} sec + +Alert text: +
{alert.text.format(*args, **kwargs)}
+''' + if time.time() - last_sent > alert.timeout: + self.send_message(text) + self.set_alert_sent(alert_name) + + def init(self): + if not self.ton.get_mode_value('alert-bot'): + return + if self.token is None or self.chat_id is None: + raise Exception("BotToken or ChatId is not set") + import telebot + self.bot = telebot.TeleBot(self.token, parse_mode="HTML") + self.hostname = get_hostname() + self.inited = True + + def set_alert_sent(self, alert_name: str): + self.ton.local.db['alerts'][alert_name] = int(time.time()) + + def get_alert_sent(self, alert_name: str): + return self.ton.local.db['alerts'].get(alert_name, 0) + + def check_db_usage(self): + usage = self.ton.GetDbUsage() + if usage > 95: + self.send_alert("db_usage_95") + elif usage > 80: + self.send_alert("db_usage_80") + + def check_validator_wallet_balance(self): + validator_wallet = self.ton.GetValidatorWallet() + validator_account = self.ton.GetAccount(validator_wallet.addrB64) + if validator_account.balance < 50: + self.send_alert("low_wallet_balance", wallet=validator_wallet.addrB64, balance=validator_account.balance) + + def check_efficiency(self): + from modules.validator import ValidatorModule + validator = ValidatorModule(self.ton, self.local).find_myself(self.ton.GetValidatorsList()) + if validator is None or validator.is_masterchain is False or validator.efficiency is None: + return + config34 = self.ton.GetConfig34() + if (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8: + return # less than 80% of round passed + if validator.efficiency < 90: + self.send_alert("low_efficiency", efficiency=validator.efficiency) + + def check_sync(self): + validator_status = self.ton.GetValidatorStatus() + if not validator_status.is_working or validator_status.out_of_sync >= 20: + self.send_alert("out_of_sync", sync=validator_status.out_of_sync) + + def check_status(self): + if not self.inited: + self.init() + + self.local.try_function(self.check_db_usage) + self.local.try_function(self.check_validator_wallet_balance) + self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed + self.local.try_function(self.check_sync) + + def add_console_commands(self, console): + ... diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 686fb657..148c07d1 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -569,6 +569,10 @@ def General(local): from modules.custom_overlays import CustomOverlayModule local.start_cycle(CustomOverlayModule(ton, local).custom_overlays, sec=60, args=()) + if ton.get_mode_value('alert-bot'): + from modules.alert_bot import AlertBotModule + local.start_cycle(AlertBotModule(ton, local).check_status, sec=1000, args=()) + thr_sleep() # end define diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 87ab360f..e90fe6bd 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3054,6 +3054,9 @@ def check_enable_mode(self, name): if name == 'liquid-staking': from mytoninstaller.settings import enable_ton_http_api enable_ton_http_api(self.local) + if name == 'alert-bot': + args = ["pip", "install", "pytelegrambotapi==4.23.0"] + subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10) def enable_mode(self, name): if name not in MODES: diff --git a/mytoncore/utils.py b/mytoncore/utils.py index 0a8bdc91..a31e299c 100644 --- a/mytoncore/utils.py +++ b/mytoncore/utils.py @@ -1,6 +1,7 @@ import base64 import json import re +import subprocess def str2b64(s): @@ -97,3 +98,6 @@ def parse_db_stats(path: str): result[s[0]] = {k: float(v) for k, v in items} return result # end define + +def get_hostname(): + return subprocess.run(["hostname", "-f"], stdout=subprocess.PIPE).stdout.decode().strip() From 0b7d6f3b9b5b2156459327f5a7f71b22cc855599 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:42:57 +0400 Subject: [PATCH 071/206] fix dataclass --- modules/alert_bot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 6118ffb1..198a79b9 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -6,7 +6,8 @@ from mytonctrl.utils import timestamp2utcdatetime -class Alert(dataclasses.dataclass): +@dataclasses.dataclass +class Alert: severity: str text: str timeout: int From 1a1d069afb640bdba04af588c8d7f1a3ac4545cc Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:47:29 +0400 Subject: [PATCH 072/206] fix alerts in db --- modules/alert_bot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 198a79b9..f7d43e06 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -105,9 +105,13 @@ def init(self): self.inited = True def set_alert_sent(self, alert_name: str): + if 'alerts' not in self.ton.local.db: + self.ton.local.db['alerts'] = {} self.ton.local.db['alerts'][alert_name] = int(time.time()) def get_alert_sent(self, alert_name: str): + if 'alerts' not in self.ton.local.db: + return 0 return self.ton.local.db['alerts'].get(alert_name, 0) def check_db_usage(self): From 3f0c559a27f522012cb06b77d1887435a004fcbc Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:53:26 +0400 Subject: [PATCH 073/206] add service_down alert --- modules/alert_bot.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index f7d43e06..3c180a62 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -50,7 +50,12 @@ class Alert: "critical", "Node is out of sync on {sync} sec.", 0 - ) + ), + "service_down": Alert( + "critical", + "validator.service is down.", + 0 + ), } @@ -129,7 +134,7 @@ def check_validator_wallet_balance(self): def check_efficiency(self): from modules.validator import ValidatorModule - validator = ValidatorModule(self.ton, self.local).find_myself(self.ton.GetValidatorsList()) + validator = ValidatorModule(self.ton, self.local).find_myself(self.ton.GetValidatorsList(fast=True)) if validator is None or validator.is_masterchain is False or validator.efficiency is None: return config34 = self.ton.GetConfig34() @@ -138,9 +143,14 @@ def check_efficiency(self): if validator.efficiency < 90: self.send_alert("low_efficiency", efficiency=validator.efficiency) + def check_validator_working(self): + validator_status = self.ton.GetValidatorStatus() + if not validator_status.is_working: + self.send_alert("service_down") + def check_sync(self): validator_status = self.ton.GetValidatorStatus() - if not validator_status.is_working or validator_status.out_of_sync >= 20: + if validator_status.is_working and validator_status.out_of_sync >= 20: self.send_alert("out_of_sync", sync=validator_status.out_of_sync) def check_status(self): @@ -150,6 +160,8 @@ def check_status(self): self.local.try_function(self.check_db_usage) self.local.try_function(self.check_validator_wallet_balance) self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed + self.local.try_function(self.check_validator_working) + self.local.try_function(self.check_validator_working) self.local.try_function(self.check_sync) def add_console_commands(self, console): From 4d6354575711461151746827399039c5aeac47eb Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:55:43 +0400 Subject: [PATCH 074/206] fix alerts timeout --- modules/alert_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 3c180a62..c73f774f 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -90,7 +90,7 @@ def send_alert(self, alert_name: str, *args, **kwargs): Hostname: {self.hostname} Time: {time_} ({int(time.time())}) Severity: {alert.severity} -Next alert of this type in: {alert.timeout} sec +Next alert of this type not earlier than: {max(alert.timeout, 1000)} sec Alert text:
{alert.text.format(*args, **kwargs)}
From 029acf05c1b7a7f988428f3077c68c922bc14f85 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 3 Oct 2024 21:57:33 +0400 Subject: [PATCH 075/206] fix double service_down alert --- modules/alert_bot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index c73f774f..b0b45384 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -161,7 +161,6 @@ def check_status(self): self.local.try_function(self.check_validator_wallet_balance) self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed self.local.try_function(self.check_validator_working) - self.local.try_function(self.check_validator_working) self.local.try_function(self.check_sync) def add_console_commands(self, console): From 1ee5e9b427dd13d8efe91ba3b7895e3230739ac6 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 7 Oct 2024 11:20:06 +0400 Subject: [PATCH 076/206] update alerts text --- modules/alert_bot.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index b0b45384..88062309 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -18,7 +18,7 @@ class Alert: ALERTS = { "low_wallet_balance": Alert( - "medium", + "low", "Validator wallet {wallet} balance is low: {balance} TON.", 18*HOUR ), @@ -43,7 +43,7 @@ class Alert: ), "zero_block_created": Alert( "critical", - "No blocks created for the last 6 hours.", + "Validator has not created any blocks in the last 6 hours.", 6*HOUR ), "out_of_sync": Alert( @@ -56,6 +56,11 @@ class Alert: "validator.service is down.", 0 ), + "adnl_connection_failed": Alert( + "high", + "ADNL connection to node failed", + 3*HOUR + ), } @@ -85,7 +90,7 @@ def send_alert(self, alert_name: str, *args, **kwargs): if alert is None: raise Exception(f"Alert {alert_name} not found") text = f''' -MyTonCtrl Alert {alert_name} +❗️ MyTonCtrl Alert {alert_name} ❗️ Hostname: {self.hostname} Time: {time_} ({int(time.time())}) From 02ca414e7f7fff9f3317612fcc348ef7ae346a80 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 8 Oct 2024 12:48:02 +0400 Subject: [PATCH 077/206] add zero_blocks_created alert --- modules/alert_bot.py | 30 +++++++++++++++++++++++------- mytoncore/mytoncore.py | 16 +++++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 88062309..2e031357 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -41,11 +41,6 @@ class Alert: """Validator efficiency is low: {efficiency}%.""", 12*HOUR ), - "zero_block_created": Alert( - "critical", - "Validator has not created any blocks in the last 6 hours.", - 6*HOUR - ), "out_of_sync": Alert( "critical", "Node is out of sync on {sync} sec.", @@ -61,6 +56,11 @@ class Alert: "ADNL connection to node failed", 3*HOUR ), + "zero_block_created": Alert( + "critical", + "Validator has not created any blocks in the last 6 hours.", + 6 * HOUR + ), } @@ -71,6 +71,7 @@ class AlertBotModule(MtcModule): def __init__(self, ton, local, *args, **kwargs): super().__init__(ton, local, *args, **kwargs) + self.validator_module = None self.inited = False self.hostname = None self.bot = None @@ -109,6 +110,8 @@ def init(self): return if self.token is None or self.chat_id is None: raise Exception("BotToken or ChatId is not set") + from modules.validator import ValidatorModule + self.validator_module = ValidatorModule(self.ton, self.local) import telebot self.bot = telebot.TeleBot(self.token, parse_mode="HTML") self.hostname = get_hostname() @@ -132,14 +135,17 @@ def check_db_usage(self): self.send_alert("db_usage_80") def check_validator_wallet_balance(self): + if not self.ton.using_validator(): + return validator_wallet = self.ton.GetValidatorWallet() validator_account = self.ton.GetAccount(validator_wallet.addrB64) if validator_account.balance < 50: self.send_alert("low_wallet_balance", wallet=validator_wallet.addrB64, balance=validator_account.balance) def check_efficiency(self): - from modules.validator import ValidatorModule - validator = ValidatorModule(self.ton, self.local).find_myself(self.ton.GetValidatorsList(fast=True)) + if not self.ton.using_validator(): + return + validator = self.validator_module.find_myself(self.ton.GetValidatorsList(fast=True)) if validator is None or validator.is_masterchain is False or validator.efficiency is None: return config34 = self.ton.GetConfig34() @@ -158,6 +164,15 @@ def check_sync(self): if validator_status.is_working and validator_status.out_of_sync >= 20: self.send_alert("out_of_sync", sync=validator_status.out_of_sync) + def check_zero_blocks_created(self): + if not self.ton.using_validator(): + return + validators = self.ton.GetValidatorsList(start=-6*HOUR, end=-60) + validator = self.validator_module.find_myself(validators) + if validator is None or validator.blocks_created > 0: + return + self.send_alert("zero_block_created") + def check_status(self): if not self.inited: self.init() @@ -166,6 +181,7 @@ def check_status(self): self.local.try_function(self.check_validator_wallet_balance) self.local.try_function(self.check_efficiency) # todo: alert if validator is going to be slashed self.local.try_function(self.check_validator_working) + self.local.try_function(self.check_zero_blocks_created) self.local.try_function(self.check_sync) def add_console_commands(self, console): diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e90fe6bd..8e691687 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2444,7 +2444,7 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict: return data #end define - def GetValidatorsList(self, past=False, fast=False): + def GetValidatorsList(self, past=False, fast=False, start=None, end=None): # Get buffer bname = "validatorsList" + str(past) buff = self.GetFunctionBuffer(bname, timeout=60) @@ -2452,13 +2452,15 @@ def GetValidatorsList(self, past=False, fast=False): return buff #end if - timestamp = get_timestamp() - end = timestamp - 60 config = self.GetConfig34() - if fast: - start = end - 1000 - else: - start = config.get("startWorkTime") + if start is None: + if fast: + start = end - 1000 + else: + start = config.get("startWorkTime") + if end is None: + timestamp = get_timestamp() + end = timestamp - 60 if past: config = self.GetConfig32() start = config.get("startWorkTime") From 67d962083e8993d6ec097a0677fa010cee220e9b Mon Sep 17 00:00:00 2001 From: Maksim Kurbatov <94808996+yungwine@users.noreply.github.com> Date: Tue, 8 Oct 2024 19:12:24 +0400 Subject: [PATCH 078/206] Revert "update complaints validation and warnings" --- modules/validator.py | 4 ++-- mytoncore/mytoncore.py | 15 ++++----------- mytonctrl/mytonctrl.py | 15 ++------------- mytonctrl/resources/translate.json | 6 +++--- 4 files changed, 11 insertions(+), 29 deletions(-) diff --git a/modules/validator.py b/modules/validator.py index 3ce5cd6f..8efbc904 100644 --- a/modules/validator.py +++ b/modules/validator.py @@ -55,7 +55,7 @@ def check_efficiency(self, args): color_print(f"Previous round time: {{yellow}}from {start_time} to {end_time}{{endc}}") if validator: if validator.is_masterchain == False: - print(f"Validator index is greater than {config32['mainValidators']} in the previous round - no efficiency data.") + print("Validator index is greater than 100 in the previous round - no efficiency data.") elif validator.get('efficiency') is None: print('Failed to get efficiency for the previous round') else: @@ -75,7 +75,7 @@ def check_efficiency(self, args): color_print(f"Current round time: {{green}}from {start_time} to {end_time}{{endc}}") if validator: if validator.is_masterchain == False: - print(f"Validator index is greater than {config34['mainValidators']} in the current round - no efficiency data.") + print("Validator index is greater than 100 in the current round - no efficiency data.") elif (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8: print("The validation round has started recently, there is not enough data yet. " "The efficiency evaluation will become more accurate towards the end of the round.") diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 87ab360f..3f3db256 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2310,7 +2310,6 @@ def get_valid_complaints(self, complaints: dict, election_id: int): continue exists = False - vload = None for item in validators_load.values(): if 'fileName' not in item: continue @@ -2320,14 +2319,14 @@ def get_valid_complaints(self, complaints: dict, election_id: int): pseudohash = pubkey + str(election_id) if pseudohash == complaint['pseudohash']: exists = True - vload = item + vid = item['id'] break if not exists: self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint info was not found, probably it's wrong", "info") continue - if vload["id"] >= config32['mainValidators']: + if vid >= config32['mainValidators']: self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint created for non masterchain validator", "info") continue @@ -2336,13 +2335,8 @@ def get_valid_complaints(self, complaints: dict, election_id: int): self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine value is {complaint['suggestedFine']} ton", "info") continue if complaint['suggestedFinePart'] != 0: # https://github.com/ton-blockchain/ton/blob/5847897b3758bc9ea85af38e7be8fc867e4c133a/lite-client/lite-client.cpp#L3709 - if vload["id"] < config32['mainValidators'] and vload["masterBlocksCreated"] + vload["workBlocksCreated"] == 0: # masterchain validator that created 0 blocks - if complaint['suggestedFinePart'] != 42949672: # (1LL << 32) / 100 - self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine part value is {complaint['suggestedFinePart']} ton", "info") - continue - else: - self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine part value is {complaint['suggestedFinePart']} ton", "info") - continue + self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint fine part value is {complaint['suggestedFinePart']} ton", "info") + continue result[complaint['pseudohash']] = complaint return result @@ -2492,7 +2486,6 @@ def GetValidatorsList(self, past=False, fast=False): validator["efficiency"] = round(validator["wr"] * 100, 2) if saveElectionEntries and adnlAddr in saveElectionEntries: validator["walletAddr"] = saveElectionEntries[adnlAddr]["walletAddr"] - validator["stake"] = saveElectionEntries[adnlAddr].get("stake") #end for # Set buffer diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 1bc78e4d..1020c5c9 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -483,22 +483,11 @@ def check_slashed(local, ton): config32 = ton.GetConfig32() save_complaints = ton.GetSaveComplaints() complaints = save_complaints.get(str(config32['startWorkTime'])) - from modules.validator import ValidatorModule - module = ValidatorModule(ton, local) - vl = ton.GetValidatorsList(past=True) - me = module.find_myself(vl) - if not me: # we were not a validator in the previous round - return if not complaints: return for c in complaints.values(): - if c["adnl"] == me["adnlAddr"] and c["isPassed"]: - if me.get("stake"): - fine = f"""{round(c['suggestedFine'] + me["stake"] * (c['suggestedFinePart'] / (1<<32)))} TON""" - else: # unknown stake amount so just print percents - fine = f"""{round(c['suggestedFine'])} TON + {(c['suggestedFinePart'] / (1<<32)) * 100} % of stake""" - warning = local.translate("slashed_warning").format(fine) - print_warning(local, warning) + if c["adnl"] == ton.GetAdnlAddr() and c["isPassed"]: + print_warning(local, "slashed_warning") #end define def check_adnl(local, ton): diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 19dfbeec..632158b4 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -445,9 +445,9 @@ "zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}" }, "slashed_warning": { - "en": "{red}You were fined by {0} for low efficiency in the previous round.{endc}", - "ru": "{red}Вы были оштрафованы на {0} за низкую эффективность в предыдущем раунде.{endc}", - "zh_TW": "{red}您因上一輪效率低而被罰款 {0}。{endc}" + "en": "{red}You were fined by 101 TON for low efficiency in the previous round.{endc}", + "ru": "{red}Вы были оштрафованы на 101 TON за низкую эффективность в предыдущем раунде.{endc}", + "zh_TW": "{red}您因上一輪效率低而被罰款 101 TON。{endc}" }, "add_custom_overlay_cmd": { "en": "Add custom overlay", From 306f7b3ddb9bfa998d0abbf4fb3260eea2e7ec8f Mon Sep 17 00:00:00 2001 From: Maksim Kurbatov <94808996+yungwine@users.noreply.github.com> Date: Tue, 8 Oct 2024 19:13:39 +0400 Subject: [PATCH 079/206] fix fine warning for testnet --- modules/validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/validator.py b/modules/validator.py index 8efbc904..3f4e56d5 100644 --- a/modules/validator.py +++ b/modules/validator.py @@ -55,7 +55,7 @@ def check_efficiency(self, args): color_print(f"Previous round time: {{yellow}}from {start_time} to {end_time}{{endc}}") if validator: if validator.is_masterchain == False: - print("Validator index is greater than 100 in the previous round - no efficiency data.") + print(f"Validator index is greater than {config32['mainValidators']} in the previous round - no efficiency data.") elif validator.get('efficiency') is None: print('Failed to get efficiency for the previous round') else: @@ -75,7 +75,7 @@ def check_efficiency(self, args): color_print(f"Current round time: {{green}}from {start_time} to {end_time}{{endc}}") if validator: if validator.is_masterchain == False: - print("Validator index is greater than 100 in the current round - no efficiency data.") + print(f"Validator index is greater than {config34['mainValidators']} in the previous round - no efficiency data.") elif (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8: print("The validation round has started recently, there is not enough data yet. " "The efficiency evaluation will become more accurate towards the end of the round.") From 4a45167d35bae7f98ffae823e7a22ce727779091 Mon Sep 17 00:00:00 2001 From: Maksim Kurbatov <94808996+yungwine@users.noreply.github.com> Date: Tue, 8 Oct 2024 19:14:39 +0400 Subject: [PATCH 080/206] fix typo --- modules/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/validator.py b/modules/validator.py index 3f4e56d5..3ce5cd6f 100644 --- a/modules/validator.py +++ b/modules/validator.py @@ -75,7 +75,7 @@ def check_efficiency(self, args): color_print(f"Current round time: {{green}}from {start_time} to {end_time}{{endc}}") if validator: if validator.is_masterchain == False: - print(f"Validator index is greater than {config34['mainValidators']} in the previous round - no efficiency data.") + print(f"Validator index is greater than {config34['mainValidators']} in the current round - no efficiency data.") elif (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8: print("The validation round has started recently, there is not enough data yet. " "The efficiency evaluation will become more accurate towards the end of the round.") From e0bb941200059c897143978aef71ca92bc487c9d Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 8 Oct 2024 19:56:21 +0400 Subject: [PATCH 081/206] add alert if validator was slashed --- modules/alert_bot.py | 13 +++++++++++++ modules/validator.py | 11 +++++++++++ mytonctrl/mytonctrl.py | 15 ++++++++------- mytonctrl/resources/translate.json | 6 +++--- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 2e031357..a513d545 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -61,6 +61,11 @@ class Alert: "Validator has not created any blocks in the last 6 hours.", 6 * HOUR ), + "validator_slashed": Alert( + "high", + "Validator has been slashed in previous round for {amount} TON", + 0 + ), } @@ -173,6 +178,13 @@ def check_zero_blocks_created(self): return self.send_alert("zero_block_created") + def check_slashed(self): + if not self.ton.using_validator(): + return + c = self.validator_module.get_my_complaint() + if c is not None: + self.send_alert("validator_slashed", amount=int(c['suggestedFine'])) + def check_status(self): if not self.inited: self.init() @@ -183,6 +195,7 @@ def check_status(self): self.local.try_function(self.check_validator_working) self.local.try_function(self.check_zero_blocks_created) self.local.try_function(self.check_sync) + self.local.try_function(self.check_slashed) def add_console_commands(self, console): ... diff --git a/modules/validator.py b/modules/validator.py index 3ce5cd6f..39317924 100644 --- a/modules/validator.py +++ b/modules/validator.py @@ -91,6 +91,17 @@ def check_efficiency(self, args): print("Couldn't find this validator in the current round") # end define + def get_my_complaint(self): + config32 = self.ton.GetConfig32() + save_complaints = self.ton.GetSaveComplaints() + complaints = save_complaints.get(str(config32['startWorkTime'])) + if not complaints: + return + for c in complaints.values(): + if c["adnl"] == self.ton.GetAdnlAddr() and c["isPassed"]: + return c + # end define + def add_console_commands(self, console): console.AddItem("vo", self.vote_offer, self.local.translate("vo_cmd")) console.AddItem("ve", self.vote_election_entry, self.local.translate("ve_cmd")) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 1020c5c9..07753126 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -480,14 +480,15 @@ def check_tg_channel(local, ton): #end difine def check_slashed(local, ton): - config32 = ton.GetConfig32() - save_complaints = ton.GetSaveComplaints() - complaints = save_complaints.get(str(config32['startWorkTime'])) - if not complaints: + validator_status = ton.GetValidatorStatus() + if not ton.using_validator() or not validator_status.is_working or validator_status.out_of_sync >= 20: return - for c in complaints.values(): - if c["adnl"] == ton.GetAdnlAddr() and c["isPassed"]: - print_warning(local, "slashed_warning") + from modules import ValidatorModule + validator_module = ValidatorModule(ton, local) + c = validator_module.get_my_complaint() + if c: + warning = local.translate("slashed_warning").format(int(c['suggestedFine'])) + print_warning(local, warning) #end define def check_adnl(local, ton): diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 632158b4..60e1501a 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -445,9 +445,9 @@ "zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}" }, "slashed_warning": { - "en": "{red}You were fined by 101 TON for low efficiency in the previous round.{endc}", - "ru": "{red}Вы были оштрафованы на 101 TON за низкую эффективность в предыдущем раунде.{endc}", - "zh_TW": "{red}您因上一輪效率低而被罰款 101 TON。{endc}" + "en": "{red}You were fined by {0} for low efficiency in the previous round.{endc}", + "ru": "{red}Вы были оштрафованы на {0} за низкую эффективность в предыдущем раунде.{endc}", + "zh_TW": "{red}您因上一輪效率低而被罰款 {0}。{endc}" }, "add_custom_overlay_cmd": { "en": "Add custom overlay", From d3c456ca33da2802e82ff36ac941a94da114e52d Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 13:35:48 +0400 Subject: [PATCH 082/206] rm telebot dependency for alert-bot mode --- modules/alert_bot.py | 20 ++++++++++++-------- mytoncore/mytoncore.py | 3 --- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index a513d545..f3faca8f 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -1,5 +1,6 @@ import dataclasses import time +import requests from modules.module import MtcModule from mytoncore import get_hostname @@ -64,7 +65,7 @@ class Alert: "validator_slashed": Alert( "high", "Validator has been slashed in previous round for {amount} TON", - 0 + 10*HOUR ), } @@ -79,15 +80,20 @@ def __init__(self, ton, local, *args, **kwargs): self.validator_module = None self.inited = False self.hostname = None - self.bot = None self.token = self.ton.local.db.get("BotToken") self.chat_id = self.ton.local.db.get("ChatId") def send_message(self, text: str): - if self.bot is not None: - self.bot.send_message(self.chat_id, text) - else: - raise Exception("send_message error: bot is not initialized") + if self.token is None: + raise Exception("send_message error: token is not initialized") + request_url = f"https://api.telegram.org/bot{self.token}/sendMessage" + data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML'} + response = requests.post(request_url, data=data, timeout=3) + if response.status_code != 200: + raise Exception(f"send_message error: {response.text}") + response = response.json() + if not response['ok']: + raise Exception(f"send_message error: {response}") def send_alert(self, alert_name: str, *args, **kwargs): last_sent = self.get_alert_sent(alert_name) @@ -117,8 +123,6 @@ def init(self): raise Exception("BotToken or ChatId is not set") from modules.validator import ValidatorModule self.validator_module = ValidatorModule(self.ton, self.local) - import telebot - self.bot = telebot.TeleBot(self.token, parse_mode="HTML") self.hostname = get_hostname() self.inited = True diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e6cfcd16..44975561 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3049,9 +3049,6 @@ def check_enable_mode(self, name): if name == 'liquid-staking': from mytoninstaller.settings import enable_ton_http_api enable_ton_http_api(self.local) - if name == 'alert-bot': - args = ["pip", "install", "pytelegrambotapi==4.23.0"] - subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=10) def enable_mode(self, name): if name not in MODES: From 4df15f50630a8eaab6623b5ece3e658e8c7f0e27 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 13:46:02 +0400 Subject: [PATCH 083/206] fix typo --- mytonctrl/resources/translate.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 60e1501a..47be11b4 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -445,9 +445,9 @@ "zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}" }, "slashed_warning": { - "en": "{red}You were fined by {0} for low efficiency in the previous round.{endc}", - "ru": "{red}Вы были оштрафованы на {0} за низкую эффективность в предыдущем раунде.{endc}", - "zh_TW": "{red}您因上一輪效率低而被罰款 {0}。{endc}" + "en": "{red}You were fined by {0} TON for low efficiency in the previous round.{endc}", + "ru": "{red}Вы были оштрафованы на {0} TON за низкую эффективность в предыдущем раунде.{endc}", + "zh_TW": "{red}您因上一輪效率低而被罰款 {0} TON。{endc}" }, "add_custom_overlay_cmd": { "en": "Add custom overlay", From b2e817ffacd31027115ec5cf49f375acbdc76c79 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 20:55:13 +0400 Subject: [PATCH 084/206] fix GetValidatorsList --- mytoncore/mytoncore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 44975561..e7155502 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2447,14 +2447,14 @@ def GetValidatorsList(self, past=False, fast=False, start=None, end=None): #end if config = self.GetConfig34() + if end is None: + timestamp = get_timestamp() + end = timestamp - 60 if start is None: if fast: start = end - 1000 else: start = config.get("startWorkTime") - if end is None: - timestamp = get_timestamp() - end = timestamp - 60 if past: config = self.GetConfig32() start = config.get("startWorkTime") From e12f9a08c7aad3df2358a433e80e3854194bb516 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 21:14:47 +0400 Subject: [PATCH 085/206] fix buffering GetValidatorsList --- mytoncore/mytoncore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e7155502..88c3e873 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2440,7 +2440,7 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False) -> dict: def GetValidatorsList(self, past=False, fast=False, start=None, end=None): # Get buffer - bname = "validatorsList" + str(past) + bname = "validatorsList" + str(past) + str(start) + str(end) buff = self.GetFunctionBuffer(bname, timeout=60) if buff: return buff From 101dd9c18120245a19376e66081399450f4b59ba Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 22:23:17 +0400 Subject: [PATCH 086/206] add constants to alerting --- modules/alert_bot.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index f3faca8f..b01e706b 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -3,6 +3,7 @@ import requests from modules.module import MtcModule +from mypylib.mypylib import get_timestamp from mytoncore import get_hostname from mytonctrl.utils import timestamp2utcdatetime @@ -15,6 +16,8 @@ class Alert: HOUR = 3600 +VALIDATION_PERIOD = 65536 +FREEZE_PERIOD = 32768 ALERTS = { @@ -40,7 +43,7 @@ class Alert: "low_efficiency": Alert( "high", """Validator efficiency is low: {efficiency}%.""", - 12*HOUR + VALIDATION_PERIOD // 3 ), "out_of_sync": Alert( "critical", @@ -59,13 +62,13 @@ class Alert: ), "zero_block_created": Alert( "critical", - "Validator has not created any blocks in the last 6 hours.", - 6 * HOUR + "Validator has not created any blocks in the last {hours} hours.", + VALIDATION_PERIOD // 3 ), "validator_slashed": Alert( "high", "Validator has been slashed in previous round for {amount} TON", - 10*HOUR + FREEZE_PERIOD ), } @@ -116,6 +119,13 @@ def send_alert(self, alert_name: str, *args, **kwargs): self.send_message(text) self.set_alert_sent(alert_name) + def set_global_vars(self): + # set global vars for correct alerts timeouts for current network + config15 = self.ton.GetConfig15() + global VALIDATION_PERIOD, FREEZE_PERIOD + VALIDATION_PERIOD = config15["validatorsElectedFor"] + FREEZE_PERIOD = config15["stakeHeldFor"] + def init(self): if not self.ton.get_mode_value('alert-bot'): return @@ -124,6 +134,7 @@ def init(self): from modules.validator import ValidatorModule self.validator_module = ValidatorModule(self.ton, self.local) self.hostname = get_hostname() + self.set_global_vars() self.inited = True def set_alert_sent(self, alert_name: str): @@ -148,7 +159,7 @@ def check_validator_wallet_balance(self): return validator_wallet = self.ton.GetValidatorWallet() validator_account = self.ton.GetAccount(validator_wallet.addrB64) - if validator_account.balance < 50: + if validator_account.balance < 10: self.send_alert("low_wallet_balance", wallet=validator_wallet.addrB64, balance=validator_account.balance) def check_efficiency(self): @@ -176,11 +187,14 @@ def check_sync(self): def check_zero_blocks_created(self): if not self.ton.using_validator(): return - validators = self.ton.GetValidatorsList(start=-6*HOUR, end=-60) + ts = get_timestamp() + period = VALIDATION_PERIOD // 3 # 6h for mainnet, 40m for testnet + start, end = ts - period, ts - 60 + validators = self.ton.GetValidatorsList(start=start, end=end) validator = self.validator_module.find_myself(validators) if validator is None or validator.blocks_created > 0: return - self.send_alert("zero_block_created") + self.send_alert("zero_block_created", hours=period // 3600) def check_slashed(self): if not self.ton.using_validator(): From 8375ed072f2dc7bce9e104e1e4922d2af1297028 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 9 Oct 2024 22:29:36 +0400 Subject: [PATCH 087/206] fix check_zero_blocks_created --- modules/alert_bot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index b01e706b..21ed63f1 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -190,6 +190,9 @@ def check_zero_blocks_created(self): ts = get_timestamp() period = VALIDATION_PERIOD // 3 # 6h for mainnet, 40m for testnet start, end = ts - period, ts - 60 + config34 = self.ton.GetConfig34() + if start < config34.startWorkTime: # round started recently + return validators = self.ton.GetValidatorsList(start=start, end=end) validator = self.validator_module.find_myself(validators) if validator is None or validator.blocks_created > 0: From de67b55d8e9afd63ef3ab4fd5aff31e67f4c0bf6 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 10 Oct 2024 15:15:30 +0400 Subject: [PATCH 088/206] return git_pool_data cmd --- modules/utilities.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/modules/utilities.py b/modules/utilities.py index 958a564a..f659b65a 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -335,6 +335,21 @@ def print_validator_list(self, args): print_table(table) # end define + def get_pool_data(self, args): + try: + pool_name = args[0] + except: + color_print("{red}Bad args. Usage:{endc} get_pool_data ") + return + if self.ton.IsAddr(pool_name): + pool_addr = pool_name + else: + pool = self.ton.GetLocalPool(pool_name) + pool_addr = pool.addrB64 + pool_data = self.ton.GetPoolData(pool_addr) + print(json.dumps(pool_data, indent=4)) + # end define + def add_console_commands(self, console): console.AddItem("vas", self.view_account_status, self.local.translate("vas_cmd")) console.AddItem("vah", self.view_account_history, self.local.translate("vah_cmd")) @@ -350,3 +365,4 @@ def add_console_commands(self, console): console.AddItem("vl", self.print_validator_list, self.local.translate("vl_cmd")) console.AddItem("cl", self.print_complaints_list, self.local.translate("cl_cmd")) + console.AddItem("get_pool_data", self.get_pool_data, self.local.translate("get_pool_data_cmd")) From 09e1e81260cf57305f64377b66f8225dbd157c65 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 10 Oct 2024 16:20:53 +0400 Subject: [PATCH 089/206] rm 'Next alert...' from alerting message --- modules/alert_bot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 21ed63f1..8a748276 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -110,7 +110,6 @@ def send_alert(self, alert_name: str, *args, **kwargs): Hostname: {self.hostname} Time: {time_} ({int(time.time())}) Severity: {alert.severity} -Next alert of this type not earlier than: {max(alert.timeout, 1000)} sec Alert text:
{alert.text.format(*args, **kwargs)}
From a27fcec182257a39bcd93b0a0336bcdbc11e6a2d Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 10 Oct 2024 16:26:57 +0400 Subject: [PATCH 090/206] add check_adnl_connection_failed --- modules/alert_bot.py | 8 ++++++++ modules/utilities.py | 34 ++++++++++++++++++++++++++++++++-- mytonctrl/mytonctrl.py | 28 +++------------------------- 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 8a748276..339f0fcd 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -205,6 +205,13 @@ def check_slashed(self): if c is not None: self.send_alert("validator_slashed", amount=int(c['suggestedFine'])) + def check_adnl_connection_failed(self): + from modules.utilities import UtilitiesModule + utils_module = UtilitiesModule(self.ton, self.local) + ok, error = utils_module.check_adnl_connection() + if not ok: + self.send_alert("adnl_connection_failed") + def check_status(self): if not self.inited: self.init() @@ -216,6 +223,7 @@ def check_status(self): self.local.try_function(self.check_zero_blocks_created) self.local.try_function(self.check_sync) self.local.try_function(self.check_slashed) + self.local.try_function(self.check_adnl_connection_failed) def add_console_commands(self, console): ... diff --git a/modules/utilities.py b/modules/utilities.py index 958a564a..a19ea581 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -1,9 +1,10 @@ -import base64 import json -import os +import random import subprocess import time +import requests + from mypylib.mypylib import color_print, print_table, color_text, timeago, bcolors from modules.module import MtcModule @@ -335,6 +336,35 @@ def print_validator_list(self, args): print_table(table) # end define + def check_adnl_connection(self): + telemetry = self.ton.local.db.get("sendTelemetry", False) + check_adnl = self.ton.local.db.get("checkAdnl", telemetry) + if not check_adnl: + return True, '' + self.local.add_log('Checking ADNL connection to local node', 'info') + hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27', + '103.106.3.171'] + hosts = random.sample(hosts, k=3) + data = self.ton.get_local_adnl_data() + error = '' + ok = True + for host in hosts: + url = f'http://{host}/adnl_check' + try: + response = requests.post(url, json=data, timeout=5).json() + except Exception as e: + ok = False + error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}' + continue + result = response.get("ok") + if result: + ok = True + break + if not result: + ok = False + error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}' + return ok, error + def add_console_commands(self, console): console.AddItem("vas", self.view_account_status, self.local.translate("vas_cmd")) console.AddItem("vah", self.view_account_history, self.local.translate("vah_cmd")) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 07753126..535de65a 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -492,31 +492,9 @@ def check_slashed(local, ton): #end define def check_adnl(local, ton): - telemetry = ton.local.db.get("sendTelemetry", False) - check_adnl = ton.local.db.get("checkAdnl", telemetry) - local.add_log('Checking ADNL connection to local node', 'info') - if not check_adnl: - return - hosts = ['45.129.96.53', '5.154.181.153', '2.56.126.137', '91.194.11.68', '45.12.134.214', '138.124.184.27', '103.106.3.171'] - hosts = random.sample(hosts, k=3) - data = ton.get_local_adnl_data() - error = '' - ok = True - for host in hosts: - url = f'http://{host}/adnl_check' - try: - response = requests.post(url, json=data, timeout=5).json() - except Exception as e: - ok = False - error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}' - continue - result = response.get("ok") - if result: - ok = True - break - if not result: - ok = False - error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}' + from modules.utilities import UtilitiesModule + utils_module = UtilitiesModule(ton, local) + ok, error = utils_module.check_adnl_connection() if not ok: print_warning(local, error) #end define From ffd8c4175908a11822eb54b8f0f7ca546ac08231 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 10 Oct 2024 16:28:31 +0400 Subject: [PATCH 091/206] round hours in zero_block_created alert --- modules/alert_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 339f0fcd..4deda90c 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -196,7 +196,7 @@ def check_zero_blocks_created(self): validator = self.validator_module.find_myself(validators) if validator is None or validator.blocks_created > 0: return - self.send_alert("zero_block_created", hours=period // 3600) + self.send_alert("zero_block_created", hours=round(period // 3600, 1)) def check_slashed(self): if not self.ton.using_validator(): From fdde23b1055384db5e5c86a7441b779d30651b44 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 11 Oct 2024 12:30:41 +0400 Subject: [PATCH 092/206] fine non-master vals that created zero blocks --- mytoncore/mytoncore.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 3f3db256..127db692 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1624,7 +1624,7 @@ def CreateWallet(self, name, workchain=0, version="v1", **kwargs): if os.path.isfile(wallet_path + ".pk") and "v3" not in version: self.local.add_log("CreateWallet error: Wallet already exists: " + name, "warning") else: - fift_args = self.get_new_wallet_fift_args(version, workchain=workchain, + fift_args = self.get_new_wallet_fift_args(version, workchain=workchain, wallet_path=wallet_path, subwallet=subwallet) result = self.fift.Run(fift_args) if "Creating new" not in result: @@ -1681,7 +1681,7 @@ def import_wallet_with_version(self, key, version, **kwargs): wallet_path = self.walletsDir + wallet_name with open(wallet_path + ".pk", 'wb') as file: file.write(pk_bytes) - fift_args = self.get_new_wallet_fift_args(version, workchain=workchain, + fift_args = self.get_new_wallet_fift_args(version, workchain=workchain, wallet_path=wallet_path, subwallet=subwallet) result = self.fift.Run(fift_args) if "Creating new" not in result: @@ -2310,6 +2310,7 @@ def get_valid_complaints(self, complaints: dict, election_id: int): continue exists = False + vload = None for item in validators_load.values(): if 'fileName' not in item: continue @@ -2319,15 +2320,16 @@ def get_valid_complaints(self, complaints: dict, election_id: int): pseudohash = pubkey + str(election_id) if pseudohash == complaint['pseudohash']: exists = True - vid = item['id'] + vload = item break if not exists: self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint info was not found, probably it's wrong", "info") continue - if vid >= config32['mainValidators']: - self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint created for non masterchain validator", "info") + if (vload["id"] >= config32['mainValidators'] and + vload["masterBlocksCreated"] + vload["workBlocksCreated"] > 0): + self.local.add_log(f"complaint {complaint['hash_hex']} declined: complaint created for non masterchain validator that created more than zero blocks", "info") continue # check complaint fine value @@ -2524,7 +2526,7 @@ def CheckValidators(self, start, end): pseudohash = pubkey + str(electionId) if pseudohash in valid_complaints or pseudohash in voted_complaints_pseudohashes: # do not create complaints that already created or voted by ourself continue - if item['id'] >= config['mainValidators']: # do not create complaints for non-masterchain validators + if item['id'] >= config['mainValidators'] and item["masterBlocksCreated"] + item["workBlocksCreated"] > 0: # create complaints for non-masterchain validators only if they created 0 blocks continue # Create complaint fileName = self.remove_proofs_from_complaint(fileName) From b47ab237d3cbfd9b4152e8d35e7c960055b1de2d Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 14 Oct 2024 19:59:38 +0400 Subject: [PATCH 093/206] add en(dis)bling alerts --- modules/alert_bot.py | 53 ++++++++++++++++++++++++++---- mytoncore/mytoncore.py | 3 ++ mytonctrl/mytonctrl.py | 5 +++ mytonctrl/resources/translate.json | 15 +++++++++ 4 files changed, 69 insertions(+), 7 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 4deda90c..8756a9c9 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -3,7 +3,7 @@ import requests from modules.module import MtcModule -from mypylib.mypylib import get_timestamp +from mypylib.mypylib import get_timestamp, print_table, color_print from mytoncore import get_hostname from mytonctrl.utils import timestamp2utcdatetime @@ -99,6 +99,8 @@ def send_message(self, text: str): raise Exception(f"send_message error: {response}") def send_alert(self, alert_name: str, *args, **kwargs): + if not self.alert_is_enabled(alert_name): + return last_sent = self.get_alert_sent(alert_name) time_ = timestamp2utcdatetime(int(time.time())) alert = ALERTS.get(alert_name) @@ -136,15 +138,50 @@ def init(self): self.set_global_vars() self.inited = True - def set_alert_sent(self, alert_name: str): + def get_alert_from_db(self, alert_name: str): if 'alerts' not in self.ton.local.db: self.ton.local.db['alerts'] = {} - self.ton.local.db['alerts'][alert_name] = int(time.time()) + if alert_name not in self.ton.local.db['alerts']: + self.ton.local.db['alerts'][alert_name] = {'sent': 0, 'enabled': True} + return self.ton.local.db['alerts'][alert_name] + + def set_alert_sent(self, alert_name: str): + alert = self.get_alert_from_db(alert_name) + alert['sent'] = int(time.time()) def get_alert_sent(self, alert_name: str): - if 'alerts' not in self.ton.local.db: - return 0 - return self.ton.local.db['alerts'].get(alert_name, 0) + alert = self.get_alert_from_db(alert_name) + return alert.get('sent', 0) + + def alert_is_enabled(self, alert_name: str): + alert = self.get_alert_from_db(alert_name) + return alert.get('enabled', True) # default is True + + def set_alert_enabled(self, alert_name: str, enabled: bool): + alert = self.get_alert_from_db(alert_name) + alert['enabled'] = enabled + self.ton.local.save() + + def enable_alert(self, args): + if len(args) != 1: + raise Exception("Usage: enable_alert ") + alert_name = args[0] + self.set_alert_enabled(alert_name, True) + color_print("enable_alert - {green}OK{endc}") + + def disable_alert(self, args): + if len(args) != 1: + raise Exception("Usage: disable_alert ") + alert_name = args[0] + self.set_alert_enabled(alert_name, False) + color_print("disable_alert - {green}OK{endc}") + + def print_alerts(self, args): + table = [['Name', 'Enabled', 'Last sent']] + for alert_name in ALERTS: + alert = self.get_alert_from_db(alert_name) + table.append([alert_name, alert['enabled'], alert['sent']]) + print_table(table) def check_db_usage(self): usage = self.ton.GetDbUsage() @@ -226,4 +263,6 @@ def check_status(self): self.local.try_function(self.check_adnl_connection_failed) def add_console_commands(self, console): - ... + console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) + console.AddItem("disable_alert", self.disable_alert, self.local.translate("disable_alert_cmd")) + console.AddItem("list_alerts", self.print_alerts, self.local.translate("list_alerts_cmd")) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 88c3e873..2b7e2119 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3089,6 +3089,9 @@ def using_validator(self): def using_liteserver(self): return self.get_mode_value('liteserver') + def using_alert_bot(self): + return self.get_mode_value('alert-bot') + def Tlb2Json(self, text): # Заменить скобки start = 0 diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 535de65a..44a868e5 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -135,6 +135,11 @@ def inject_globals(func): module = ControllerModule(ton, local) module.add_console_commands(console) + if ton.using_alert_bot(): + from modules.alert_bot import AlertBotModule + module = AlertBotModule(ton, local) + module.add_console_commands(console) + console.AddItem("cleanup", inject_globals(cleanup_validator_db), local.translate("cleanup_cmd")) console.AddItem("benchmark", inject_globals(run_benchmark), local.translate("benchmark_cmd")) # console.AddItem("activate_ton_storage_provider", inject_globals(activate_ton_storage_provider), local.translate("activate_ton_storage_provider_cmd")) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 47be11b4..2a5ae89e 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -464,6 +464,21 @@ "ru": "Удалить пользовательский оверлей", "zh_TW": "刪除自定義覆蓋" }, + "enable_alert_cmd": { + "en": "Enable specific Telegram Bot alert", + "ru": "Включить определенное оповещение через Telegram Bot", + "zh_TW": "啟用特定的 Telegram Bot 警報" + }, + "disable_alert_cmd": { + "en": "Disable specific Telegram Bot alert", + "ru": "Отключить определенное оповещение через Telegram Bot", + "zh_TW": "禁用特定的 Telegram Bot 警報" + }, + "list_alerts_cmd": { + "en": "List all available Telegram Bot alerts", + "ru": "Список всех доступных оповещений через Telegram Bot", + "zh_TW": "列出所有可用的 Telegram Bot 警報" + }, "cleanup_cmd": { "en": "Clean node old logs and temp files", "ru": "Очистить старые логи и временные файлы ноды", From bf84a530c0b94e6097d0b574cc886d9767ca8765 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 17 Oct 2024 21:40:44 +0400 Subject: [PATCH 094/206] add test_alert cmd --- modules/alert_bot.py | 6 ++++++ mytonctrl/resources/translate.json | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 8756a9c9..5ba4d99a 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -89,6 +89,8 @@ def __init__(self, ton, local, *args, **kwargs): def send_message(self, text: str): if self.token is None: raise Exception("send_message error: token is not initialized") + if self.chat_id is None: + raise Exception("send_message error: chat_id is not initialized") request_url = f"https://api.telegram.org/bot{self.token}/sendMessage" data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML'} response = requests.post(request_url, data=data, timeout=3) @@ -183,6 +185,9 @@ def print_alerts(self, args): table.append([alert_name, alert['enabled'], alert['sent']]) print_table(table) + def test_alert(self, args): + self.send_message('Test alert') + def check_db_usage(self): usage = self.ton.GetDbUsage() if usage > 95: @@ -266,3 +271,4 @@ def add_console_commands(self, console): console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) console.AddItem("disable_alert", self.disable_alert, self.local.translate("disable_alert_cmd")) console.AddItem("list_alerts", self.print_alerts, self.local.translate("list_alerts_cmd")) + console.AddItem("test_alert", self.test_alert, self.local.translate("test_alert_cmd")) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 2a5ae89e..929eea8a 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -479,6 +479,11 @@ "ru": "Список всех доступных оповещений через Telegram Bot", "zh_TW": "列出所有可用的 Telegram Bot 警報" }, + "test_alert_cmd": { + "en": "Send test alert via Telegram Bot", + "ru": "Отправить тестовое оповещение через Telegram Bot", + "zh_TW": "通過 Telegram Bot 發送測試警報" + }, "cleanup_cmd": { "en": "Clean node old logs and temp files", "ru": "Очистить старые логи и временные файлы ноды", From a4c9fb8a1e79ce45c1861e7769a97baecdb376fe Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Mon, 21 Oct 2024 20:53:18 +0300 Subject: [PATCH 095/206] bugfix --- mytonctrl/mytonctrl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 44a868e5..4b3dd433 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -527,6 +527,9 @@ def mode_status(ton, args): table = [["Name", "Status", "Description"]] for mode_name in modes: mode = get_mode(mode_name) + if mode is None: + color_print(f"{{red}}Mode {mode_name} not found{{endc}}") + continue status = color_text('{green}enabled{endc}' if modes[mode_name] else '{red}disabled{endc}') table.append([mode_name, status, mode.description]) print_table(table) From c51e00af6f16495c4040528635598fd7fdaba5c6 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 22 Oct 2024 18:06:32 +0400 Subject: [PATCH 096/206] fix low_efficiency alert --- modules/alert_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 5ba4d99a..6487bdaf 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -206,7 +206,7 @@ def check_validator_wallet_balance(self): def check_efficiency(self): if not self.ton.using_validator(): return - validator = self.validator_module.find_myself(self.ton.GetValidatorsList(fast=True)) + validator = self.validator_module.find_myself(self.ton.GetValidatorsList()) if validator is None or validator.is_masterchain is False or validator.efficiency is None: return config34 = self.ton.GetConfig34() From 59f2f379cecfaa1715600a21236aa86e9e72698c Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 22 Oct 2024 18:22:32 +0400 Subject: [PATCH 097/206] print efficiency for non-master validators if its 0 --- modules/alert_bot.py | 5 ++++- modules/validator.py | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 6487bdaf..09ec0753 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -207,11 +207,14 @@ def check_efficiency(self): if not self.ton.using_validator(): return validator = self.validator_module.find_myself(self.ton.GetValidatorsList()) - if validator is None or validator.is_masterchain is False or validator.efficiency is None: + if validator is None or validator.efficiency is None: return config34 = self.ton.GetConfig34() if (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8: return # less than 80% of round passed + if validator.is_masterchain is False: + if validator.efficiency != 0: + return if validator.efficiency < 90: self.send_alert("low_efficiency", efficiency=validator.efficiency) diff --git a/modules/validator.py b/modules/validator.py index 39317924..a0fdfa9c 100644 --- a/modules/validator.py +++ b/modules/validator.py @@ -54,10 +54,10 @@ def check_efficiency(self, args): end_time = timestamp2utcdatetime(config32.endWorkTime) color_print(f"Previous round time: {{yellow}}from {start_time} to {end_time}{{endc}}") if validator: - if validator.is_masterchain == False: - print(f"Validator index is greater than {config32['mainValidators']} in the previous round - no efficiency data.") - elif validator.get('efficiency') is None: + if validator.get('efficiency') is None: print('Failed to get efficiency for the previous round') + elif validator.is_masterchain is False and validator.get('efficiency') != 0: + print(f"Validator index is greater than {config32['mainValidators']} in the previous round - no efficiency data.") else: efficiency = 100 if validator.efficiency > 100 else validator.efficiency color_efficiency = GetColorInt(efficiency, 90, logic="more", ending="%") @@ -74,7 +74,7 @@ def check_efficiency(self, args): end_time = timestamp2utcdatetime(int(get_timestamp())) color_print(f"Current round time: {{green}}from {start_time} to {end_time}{{endc}}") if validator: - if validator.is_masterchain == False: + if validator.is_masterchain is False and validator.efficiency != 0: print(f"Validator index is greater than {config34['mainValidators']} in the current round - no efficiency data.") elif (time.time() - config34.startWorkTime) / (config34.endWorkTime - config34.startWorkTime) < 0.8: print("The validation round has started recently, there is not enough data yet. " From f94a7dc72a9c72276a54a11082335af461cbbe02 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 24 Oct 2024 20:01:10 +0400 Subject: [PATCH 098/206] fix slashed_warning --- mytonctrl/resources/translate.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 929eea8a..22915e59 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -445,9 +445,9 @@ "zh_TW": "{red}錯誤 - 驗證器的 UDP 端口無法從外部訪問.{endc}" }, "slashed_warning": { - "en": "{red}You were fined by {0} TON for low efficiency in the previous round.{endc}", - "ru": "{red}Вы были оштрафованы на {0} TON за низкую эффективность в предыдущем раунде.{endc}", - "zh_TW": "{red}您因上一輪效率低而被罰款 {0} TON。{endc}" + "en": "{{red}}You were fined by {0} TON for low efficiency in the previous round.{{endc}}", + "ru": "{{red}}Вы были оштрафованы на {0} TON за низкую эффективность в предыдущем раунде.{{endc}}", + "zh_TW": "{{red}}您因上一輪效率低而被罰款 {0} TON。{{endc}}" }, "add_custom_overlay_cmd": { "en": "Add custom overlay", From 79bf05b3809729bc22484d5f2e3f92204bc1f6f4 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 24 Oct 2024 20:03:25 +0400 Subject: [PATCH 099/206] update alerting thread time --- modules/alert_bot.py | 6 ++++++ mytoncore/functions.py | 5 ++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 09ec0753..c618dae0 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -85,6 +85,7 @@ def __init__(self, ton, local, *args, **kwargs): self.hostname = None self.token = self.ton.local.db.get("BotToken") self.chat_id = self.ton.local.db.get("ChatId") + self.last_db_check = None def send_message(self, text: str): if self.token is None: @@ -189,6 +190,9 @@ def test_alert(self, args): self.send_message('Test alert') def check_db_usage(self): + if time.time() - self.last_db_check < 600: + return + self.last_db_check = time.time() usage = self.ton.GetDbUsage() if usage > 95: self.send_alert("db_usage_95") @@ -258,6 +262,8 @@ def check_adnl_connection_failed(self): self.send_alert("adnl_connection_failed") def check_status(self): + if not self.ton.using_alert_bot(): + return if not self.inited: self.init() diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 148c07d1..0dd136ff 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -569,9 +569,8 @@ def General(local): from modules.custom_overlays import CustomOverlayModule local.start_cycle(CustomOverlayModule(ton, local).custom_overlays, sec=60, args=()) - if ton.get_mode_value('alert-bot'): - from modules.alert_bot import AlertBotModule - local.start_cycle(AlertBotModule(ton, local).check_status, sec=1000, args=()) + from modules.alert_bot import AlertBotModule + local.start_cycle(AlertBotModule(ton, local).check_status, sec=60, args=()) thr_sleep() # end define From 9ff2b25e49ed33de7261b120a516cb436a426d68 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 25 Oct 2024 19:27:07 +0400 Subject: [PATCH 100/206] create local config on enabling tha --- mytoninstaller/mytoninstaller.py | 1 - mytoninstaller/settings.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index c5e60012..270afe97 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -194,7 +194,6 @@ def PrintLiteServerConfig(local, args): def CreateLocalConfigFile(local, args): initBlock = GetInitBlock() initBlock_b64 = dict2b64(initBlock) - user = local.buffer["user"] args = ["python3", "-m", "mytoninstaller", "-u", local.buffer.user, "-e", "clc", "-i", initBlock_b64] run_as_root(args) #end define diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 76e92908..cba219ea 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -462,6 +462,9 @@ def EnableJsonRpc(local): def enable_ton_http_api(local): local.add_log("start EnableTonHttpApi function", "debug") + if not os.path.exists('/usr/bin/ton/local.config.json'): + from mytoninstaller.mytoninstaller import CreateLocalConfigFile + CreateLocalConfigFile(local, []) ton_http_api_installer_path = pkg_resources.resource_filename('mytoninstaller.scripts', 'ton_http_api_installer.sh') exit_code = run_as_root(["bash", ton_http_api_installer_path]) if exit_code == 0: From 2777801e06260dca0a1791b605321bdb10ddcfc8 Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Sat, 26 Oct 2024 23:19:07 +0300 Subject: [PATCH 101/206] bugfix --- modules/alert_bot.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index c618dae0..e9dafed3 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -83,9 +83,9 @@ def __init__(self, ton, local, *args, **kwargs): self.validator_module = None self.inited = False self.hostname = None - self.token = self.ton.local.db.get("BotToken") - self.chat_id = self.ton.local.db.get("ChatId") - self.last_db_check = None + self.token = None + self.chat_id = None + self.last_db_check = 0 def send_message(self, text: str): if self.token is None: @@ -133,6 +133,8 @@ def set_global_vars(self): def init(self): if not self.ton.get_mode_value('alert-bot'): return + self.token = self.ton.local.db.get("BotToken") + self.chat_id = self.ton.local.db.get("ChatId") if self.token is None or self.chat_id is None: raise Exception("BotToken or ChatId is not set") from modules.validator import ValidatorModule From 1441d6724c241ed33ff2904df42e5553561e00f5 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 29 Oct 2024 12:20:30 +0400 Subject: [PATCH 102/206] increase out_of_sync and service_down alerts timeouts --- modules/alert_bot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index c618dae0..d542c40f 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -48,12 +48,12 @@ class Alert: "out_of_sync": Alert( "critical", "Node is out of sync on {sync} sec.", - 0 + 300 ), "service_down": Alert( "critical", "validator.service is down.", - 0 + 300 ), "adnl_connection_failed": Alert( "high", From 9201a7d8f9e8e965cfe6444f7d404cb4c00c5a89 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 29 Oct 2024 18:41:22 +0400 Subject: [PATCH 103/206] fix db backup restoring --- mytoncore/mytoncore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 76cf58cd..2cdbde96 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -115,6 +115,7 @@ def CheckConfigFile(self, fift, liteClient): self.local.add_log("Restoring the configuration file", "info") args = ["cp", backup_path, mconfig_path] subprocess.run(args) + self.dbFile = mconfig_path self.Refresh() elif os.path.isfile(backup_path) == False: self.local.add_log("Create backup config file", "info") From 683329d28c38286382bacf500b006cae1bd3cde8 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 30 Oct 2024 14:46:26 +0400 Subject: [PATCH 104/206] rm deposit_to_pool and withdraw_from_pool cmds from single-nominator --- modules/nominator_pool.py | 38 ++++++++++++++++++++++++++++++++++++++ modules/pool.py | 38 -------------------------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/modules/nominator_pool.py b/modules/nominator_pool.py index 14f4fbdd..d00708b0 100644 --- a/modules/nominator_pool.py +++ b/modules/nominator_pool.py @@ -87,7 +87,45 @@ def update_validator_set(self, args): self.ton.PoolUpdateValidatorSet(pool_addr, wallet) color_print("UpdateValidatorSet - {green}OK{endc}") + def do_deposit_to_pool(self, pool_addr, amount): + wallet = self.ton.GetValidatorWallet() + bocPath = self.ton.local.buffer.my_temp_dir + wallet.name + "validator-deposit-query.boc" + fiftScript = self.ton.contractsDir + "nominator-pool/func/validator-deposit.fif" + args = [fiftScript, bocPath] + result = self.ton.fift.Run(args) + resultFilePath = self.ton.SignBocWithWallet(wallet, bocPath, pool_addr, amount) + self.ton.SendFile(resultFilePath, wallet) + + def deposit_to_pool(self, args): + try: + poll_addr = args[0] + amount = float(args[1]) + except: + color_print("{red}Bad args. Usage:{endc} deposit_to_pool ") + return + self.do_deposit_to_pool(poll_addr, amount) + color_print("DepositToPool - {green}OK{endc}") + + def do_withdraw_from_pool(self, pool_addr, amount): + pool_data = self.ton.GetPoolData(pool_addr) + if pool_data["state"] == 0: + self.ton.WithdrawFromPoolProcess(pool_addr, amount) + else: + self.ton.PendWithdrawFromPool(pool_addr, amount) + + def withdraw_from_pool(self, args): + try: + pool_addr = args[0] + amount = float(args[1]) + except: + color_print("{red}Bad args. Usage:{endc} withdraw_from_pool ") + return + self.do_withdraw_from_pool(pool_addr, amount) + color_print("WithdrawFromPool - {green}OK{endc}") + def add_console_commands(self, console): console.AddItem("new_pool", self.new_pool, self.local.translate("new_pool_cmd")) console.AddItem("activate_pool", self.activate_pool, self.local.translate("activate_pool_cmd")) console.AddItem("update_validator_set", self.update_validator_set, self.local.translate("update_validator_set_cmd")) + console.AddItem("withdraw_from_pool", self.withdraw_from_pool, self.local.translate("withdraw_from_pool_cmd")) + console.AddItem("deposit_to_pool", self.deposit_to_pool, self.local.translate("deposit_to_pool_cmd")) diff --git a/modules/pool.py b/modules/pool.py index ba37b16c..16b481c5 100644 --- a/modules/pool.py +++ b/modules/pool.py @@ -58,45 +58,7 @@ def check_download_pool_contract_scripts(self): if not os.path.isdir(contract_path): self.ton.DownloadContract("https://github.com/ton-blockchain/nominator-pool") - def do_deposit_to_pool(self, pool_addr, amount): - wallet = self.ton.GetValidatorWallet() - bocPath = self.ton.local.buffer.my_temp_dir + wallet.name + "validator-deposit-query.boc" - fiftScript = self.ton.contractsDir + "nominator-pool/func/validator-deposit.fif" - args = [fiftScript, bocPath] - result = self.ton.fift.Run(args) - resultFilePath = self.ton.SignBocWithWallet(wallet, bocPath, pool_addr, amount) - self.ton.SendFile(resultFilePath, wallet) - - def deposit_to_pool(self, args): - try: - poll_addr = args[0] - amount = float(args[1]) - except: - color_print("{red}Bad args. Usage:{endc} deposit_to_pool ") - return - self.do_deposit_to_pool(poll_addr, amount) - color_print("DepositToPool - {green}OK{endc}") - - def do_withdraw_from_pool(self, pool_addr, amount): - pool_data = self.ton.GetPoolData(pool_addr) - if pool_data["state"] == 0: - self.ton.WithdrawFromPoolProcess(pool_addr, amount) - else: - self.ton.PendWithdrawFromPool(pool_addr, amount) - - def withdraw_from_pool(self, args): - try: - pool_addr = args[0] - amount = float(args[1]) - except: - color_print("{red}Bad args. Usage:{endc} withdraw_from_pool ") - return - self.do_withdraw_from_pool(pool_addr, amount) - color_print("WithdrawFromPool - {green}OK{endc}") - def add_console_commands(self, console): console.AddItem("pools_list", self.print_pools_list, self.local.translate("pools_list_cmd")) console.AddItem("delete_pool", self.delete_pool, self.local.translate("delete_pool_cmd")) console.AddItem("import_pool", self.import_pool, self.local.translate("import_pool_cmd")) - console.AddItem("deposit_to_pool", self.deposit_to_pool, self.local.translate("deposit_to_pool_cmd")) - console.AddItem("withdraw_from_pool", self.withdraw_from_pool, self.local.translate("withdraw_from_pool_cmd")) From d4d1b4616104524d22f0397d1cc335f016b847c8 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 30 Oct 2024 15:16:53 +0400 Subject: [PATCH 105/206] add stake to vl --- modules/utilities.py | 5 +++-- mytoncore/mytoncore.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/utilities.py b/modules/utilities.py index da108152..62c83efb 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -313,13 +313,14 @@ def print_validator_list(self, args): print(text) else: table = list() - table += [["id", "ADNL", "Pubkey", "Wallet", "Efficiency", "Online"]] + table += [["id", "ADNL", "Pubkey", "Wallet", "Stake", "Efficiency", "Online"]] for i, item in enumerate(data): adnl = item.get("adnlAddr") pubkey = item.get("pubkey") walletAddr = item.get("walletAddr") efficiency = item.get("efficiency") online = item.get("online") + stake = item.get("stake") if "adnl" not in args: adnl = self.reduct(adnl) if "pubkey" not in args: @@ -332,7 +333,7 @@ def print_validator_list(self, args): online = bcolors.green_text("true") if not online: online = bcolors.red_text("false") - table += [[str(i), adnl, pubkey, walletAddr, efficiency, online]] + table += [[str(i), adnl, pubkey, walletAddr, stake, efficiency, online]] print_table(table) # end define diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 76cf58cd..4c4491fa 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2490,6 +2490,7 @@ def GetValidatorsList(self, past=False, fast=False, start=None, end=None): validator["efficiency"] = round(validator["wr"] * 100, 2) if saveElectionEntries and adnlAddr in saveElectionEntries: validator["walletAddr"] = saveElectionEntries[adnlAddr]["walletAddr"] + validator["stake"] = saveElectionEntries[adnlAddr].get("stake") #end for # Set buffer From 0fbbba9488c288e4bae22bedb95b4c20d945ba89 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 11 Nov 2024 13:24:30 +0400 Subject: [PATCH 106/206] set pip libraries version --- mytoninstaller/scripts/jsonrpcinstaller.sh | 2 +- mytoninstaller/scripts/ton_http_api_installer.sh | 2 +- requirements.txt | 11 +++++------ scripts/install.sh | 2 +- scripts/ton_installer.sh | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/mytoninstaller/scripts/jsonrpcinstaller.sh b/mytoninstaller/scripts/jsonrpcinstaller.sh index e6338480..926d61e1 100755 --- a/mytoninstaller/scripts/jsonrpcinstaller.sh +++ b/mytoninstaller/scripts/jsonrpcinstaller.sh @@ -28,7 +28,7 @@ ENDC='\033[0m' # Установка компонентов python3 echo -e "${COLOR}[1/4]${ENDC} Installing required packages" -pip3 install Werkzeug json-rpc cloudscraper pyotp +pip3 install Werkzeug json-rpc cloudscraper pyotp jsonpickle # todo: set versions # Клонирование репозиториев с github.com echo -e "${COLOR}[2/4]${ENDC} Cloning github repository" diff --git a/mytoninstaller/scripts/ton_http_api_installer.sh b/mytoninstaller/scripts/ton_http_api_installer.sh index fa87c42c..a5eab69c 100644 --- a/mytoninstaller/scripts/ton_http_api_installer.sh +++ b/mytoninstaller/scripts/ton_http_api_installer.sh @@ -12,7 +12,7 @@ COLOR='\033[92m' ENDC='\033[0m' # install python3 packages -pip3 install virtualenv +pip3 install virtualenv==20.27.1 # prepare the virtual environment echo -e "${COLOR}[1/4]${ENDC} Preparing the virtual environment" diff --git a/requirements.txt b/requirements.txt index f7705c47..01f7a310 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -crc16 -requests -psutil -fastcrc -jsonpickle -pynacl +crc16==0.1.1 +requests==2.32.3 +psutil==6.1.0 +fastcrc==0.3.2 +pynacl==1.5.0 diff --git a/scripts/install.sh b/scripts/install.sh index dffe1134..1da6e42c 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -72,7 +72,7 @@ done if [ "${mode}" = "" ]; then # no mode echo "Running cli installer" wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/install.py - pip3 install inquirer + pip3 install inquirer==3.4.0 python3 install.py exit fi diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index 3ae64874..6d4ad84a 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -92,7 +92,7 @@ else fi # Установка компонентов python3 -pip3 install psutil crc16 requests +pip3 install psutil==6.1.0 crc16==0.1.1 requests==2.32.3 # build openssl 3.0 echo -e "${COLOR}[2/6]${ENDC} Building OpenSSL 3.0" From e44a0558b740372bf5174704ad998024d0402c47 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 11 Nov 2024 13:43:07 +0400 Subject: [PATCH 107/206] fix test_alert --- modules/alert_bot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index d542c40f..aea45732 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -187,6 +187,8 @@ def print_alerts(self, args): print_table(table) def test_alert(self, args): + if not self.inited: + self.init() self.send_message('Test alert') def check_db_usage(self): From f551af250e0fc08f3bee60db7bc7fe33ce60e064 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 12 Nov 2024 12:40:33 +0400 Subject: [PATCH 108/206] do not restart node on creating backups --- mytonctrl/mytonctrl.py | 2 +- mytonctrl/scripts/create_backup.sh | 14 +++++++------- mytonctrl/scripts/restore_backup.sh | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 4b3dd433..de3d7ce8 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -944,7 +944,7 @@ def create_backup(local, ton, args): color_print("{red}Bad args. Usage:{endc} create_backup [path_to_archive] [-y]") return if '-y' not in args: - res = input(f'Node and Mytoncore services will be stopped for few seconds while backup is created, Proceed [y/n]?') + res = input(f'Mytoncore service will be stopped for few seconds while backup is created, Proceed [y/n]?') if res.lower() != 'y': print('aborted.') return diff --git a/mytonctrl/scripts/create_backup.sh b/mytonctrl/scripts/create_backup.sh index 9bab1355..0c88788a 100644 --- a/mytonctrl/scripts/create_backup.sh +++ b/mytonctrl/scripts/create_backup.sh @@ -16,28 +16,28 @@ done COLOR='\033[92m' ENDC='\033[0m' -systemctl stop validator systemctl stop mytoncore -echo -e "${COLOR}[1/4]${ENDC} Stopped validator and mytoncore" +echo -e "${COLOR}[1/4]${ENDC} Stopped mytoncore service" tmp_dir="/tmp/mytoncore/backup" rm -rf $tmp_dir mkdir $tmp_dir +mkdir $tmp_dir/db -cp /var/ton-work/db/config.json ${tmp_dir} -cp -r /var/ton-work/db/keyring ${tmp_dir} +cp /var/ton-work/db/config.json ${tmp_dir}/db +cp -r /var/ton-work/db/keyring ${tmp_dir}/db cp -r /var/ton-work/keys ${tmp_dir} cp -r $mtc_dir $tmp_dir -echo -e "${COLOR}[2/4]${ENDC} Copied files to ${tmp_dir}" +python3 -c "import json;f=open('${tmp_dir}/db/config.json');json.load(f);f.close()" || exit 1 # Check if config.json is copied correctly +echo -e "${COLOR}[2/4]${ENDC} Copied files to ${tmp_dir}" -systemctl start validator systemctl start mytoncore -echo -e "${COLOR}[3/4]${ENDC} Started validator and mytoncore" +echo -e "${COLOR}[3/4]${ENDC} Started mytoncore service" tar -zcf $dest -C $tmp_dir . diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index 361c7ff9..9f0ea465 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -30,8 +30,7 @@ mkdir $tmp_dir tar -xvzf $name -C $tmp_dir rm -rf /var/ton-work/db/keyring -cp -f ${tmp_dir}/config.json /var/ton-work/db/ -cp -rf ${tmp_dir}/keyring /var/ton-work/db/ +cp -rf ${tmp_dir}/db /var/ton-work cp -rf ${tmp_dir}/keys /var/ton-work cp -rfT ${tmp_dir}/mytoncore $mtc_dir From 5837b57b70144c250dcf3161d5589fcf4b601976 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 12 Nov 2024 22:37:34 +0400 Subject: [PATCH 109/206] raise error if old backup --- mytonctrl/scripts/restore_backup.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index 9f0ea465..0d580efe 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -29,6 +29,15 @@ rm -rf $tmp_dir mkdir $tmp_dir tar -xvzf $name -C $tmp_dir +if [ ! -d ${tmp_dir}/db ]; then + echo "Error: db/ directory not found in archive. Aborting" + + systemctl start validator + systemctl start mytoncore + + exit 1 +fi + rm -rf /var/ton-work/db/keyring cp -rf ${tmp_dir}/db /var/ton-work cp -rf ${tmp_dir}/keys /var/ton-work From 56ef77c9a4fd25dea9387634a0bee6461f1e9062 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 13 Nov 2024 17:10:13 +0400 Subject: [PATCH 110/206] patch old backup --- mytonctrl/scripts/restore_backup.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index 0d580efe..ca53f0de 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -30,12 +30,11 @@ mkdir $tmp_dir tar -xvzf $name -C $tmp_dir if [ ! -d ${tmp_dir}/db ]; then - echo "Error: db/ directory not found in archive. Aborting" + echo "Old version of backup detected" + mkdir ${tmp_dir}/db + mv ${tmp_dir}/config.json ${tmp_dir}/db + mv ${tmp_dir}/keyring ${tmp_dir}/db - systemctl start validator - systemctl start mytoncore - - exit 1 fi rm -rf /var/ton-work/db/keyring From 875f9a2bd2c1771e61038946a7ee0686bd487acc Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 18 Nov 2024 14:17:20 +0400 Subject: [PATCH 111/206] round validator stake info in vl --- mytoncore/mytoncore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 4c4491fa..5aca1b9b 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -2491,6 +2491,7 @@ def GetValidatorsList(self, past=False, fast=False, start=None, end=None): if saveElectionEntries and adnlAddr in saveElectionEntries: validator["walletAddr"] = saveElectionEntries[adnlAddr]["walletAddr"] validator["stake"] = saveElectionEntries[adnlAddr].get("stake") + validator["stake"] = int(validator["stake"]) if validator["stake"] else None #end for # Set buffer From feef0cbad71a2233b003f325d03f39cfe43d0bf2 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 19 Nov 2024 22:39:54 +0400 Subject: [PATCH 112/206] add clear backup scripts --- support/clear_backup.sh | 48 ++++++++++++++++++++++++++++++++ support/setup_cleared_backup.py | 49 +++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 support/clear_backup.sh create mode 100644 support/setup_cleared_backup.py diff --git a/support/clear_backup.sh b/support/clear_backup.sh new file mode 100644 index 00000000..52409eb2 --- /dev/null +++ b/support/clear_backup.sh @@ -0,0 +1,48 @@ +name="backup.tar.gz" +dest="cleared_backup_$(hostname)_$(date +%s).tar.gz" +ton_db="" +tmp_dir="tmp/backup" +user=$(logname) + + +# Get arguments +while getopts n:d:t: flag +do + case "${flag}" in + n) name=${OPTARG};; + d) dest=${OPTARG};; + t) ton_db=${OPTARG};; + *) + echo "Flag -${flag} is not recognized. Aborting" + exit 1 ;; + esac +done + +rm -rf $tmp_dir +mkdir tmp/backup -p + +if [ ! -z "$ton_db" ]; then + mkdir ${tmp_dir}/db + cp -r "$ton_db"/db/keyring ${tmp_dir}/db + cp "$ton_db"/db/config.json ${tmp_dir}/db +else + tar -xzf $name -C $tmp_dir +fi + +rm -rf ${tmp_dir}/mytoncore +rm -rf ${tmp_dir}/keys +mv ${tmp_dir}/db/keyring ${tmp_dir}/db/old_keyring +mkdir ${tmp_dir}/db/keyring + +keys=$(python3 -c "import json;import base64;f=open('${tmp_dir}/db/config.json');config=json.load(f);f.close();keys=set();[([keys.add(base64.b64decode(key['key']).hex().upper()) for key in v['temp_keys']], [keys.add(base64.b64decode(adnl['id']).hex().upper()) for adnl in v['adnl_addrs']]) for v in config['validators']];print('\n'.join(list(keys)))") + +for key in $keys; do + mv ${tmp_dir}/db/old_keyring/${key} ${tmp_dir}/db/keyring +done + +rm -rf ${tmp_dir}/db/old_keyring + +tar -zcf $dest -C $tmp_dir . +chown $user:$user $dest + +echo -e "Cleared backup successfully created in ${dest}!" diff --git a/support/setup_cleared_backup.py b/support/setup_cleared_backup.py new file mode 100644 index 00000000..d6f8eb55 --- /dev/null +++ b/support/setup_cleared_backup.py @@ -0,0 +1,49 @@ +import argparse +import base64 +import json +import subprocess +tmp_dir = "tmp/cleared_backup" + + +def b64tohex(b64str: str): + return base64.b64decode(b64str).hex().upper() + + +def run_vc(cmd: str): + args = ['/usr/bin/ton/validator-engine-console/validator-engine-console', '-k', '/var/ton-work/keys/client', '-p', '/var/ton-work/keys/server.pub', '-a', vc_address, '--cmd', cmd] + subprocess.run(args) + + +parser = argparse.ArgumentParser() +parser.add_argument('-n') +parser.add_argument('-a') + +args = parser.parse_args() +name = args.n +vc_address = args.a + +if not name or not vc_address: + print("Usage: setup_cleared_backup.py -n -a ") + exit(1) + + +subprocess.run(f"rm -rf {tmp_dir}", shell=True) +subprocess.run(f"mkdir -p {tmp_dir}", shell=True) + +subprocess.run(f'tar -xzf {name} -C {tmp_dir}', shell=True) + +subprocess.run(f'cp -rf {tmp_dir}/db/keyring /var/ton-work/db/', shell=True) +subprocess.run(f'chown -R validator:validator /var/ton-work/db/keyring', shell=True) + +with open(f'{tmp_dir}/db/config.json', 'r') as f: + config = json.load(f) + +for v in config['validators']: + run_vc(f'addpermkey {b64tohex(v["id"])} {v["election_date"]} {v["expire_at"]}') + for tkey in v['temp_keys']: + run_vc(f'addtempkey {b64tohex(v["id"])} {b64tohex(tkey["key"])} {v["expire_at"]}') + for adnl in v['adnl_addrs']: + run_vc(f'addadnl {b64tohex(adnl["id"])} 0') + run_vc(f'addvalidatoraddr {b64tohex(v["id"])} {b64tohex(adnl["id"])} {v["expire_at"]}') + +subprocess.run(f'systemctl restart validator', shell=True) From 99a72138251a2b10ec1ab2333fa3a5e49df6dcea Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 21 Nov 2024 18:09:31 +0800 Subject: [PATCH 113/206] update scripts name --- support/{clear_backup.sh => extract_backup_node_keys.sh} | 2 +- support/{setup_cleared_backup.py => inject_backup_node_keys.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename support/{clear_backup.sh => extract_backup_node_keys.sh} (95%) rename support/{setup_cleared_backup.py => inject_backup_node_keys.py} (94%) diff --git a/support/clear_backup.sh b/support/extract_backup_node_keys.sh similarity index 95% rename from support/clear_backup.sh rename to support/extract_backup_node_keys.sh index 52409eb2..adc4448c 100644 --- a/support/clear_backup.sh +++ b/support/extract_backup_node_keys.sh @@ -45,4 +45,4 @@ rm -rf ${tmp_dir}/db/old_keyring tar -zcf $dest -C $tmp_dir . chown $user:$user $dest -echo -e "Cleared backup successfully created in ${dest}!" +echo -e "Node keys backup successfully created in ${dest}!" diff --git a/support/setup_cleared_backup.py b/support/inject_backup_node_keys.py similarity index 94% rename from support/setup_cleared_backup.py rename to support/inject_backup_node_keys.py index d6f8eb55..9566c788 100644 --- a/support/setup_cleared_backup.py +++ b/support/inject_backup_node_keys.py @@ -23,7 +23,7 @@ def run_vc(cmd: str): vc_address = args.a if not name or not vc_address: - print("Usage: setup_cleared_backup.py -n -a ") + print("Usage: inject_backup_node_keys.py -n -a ") exit(1) From 5ac0501f67cceeb4d7ed0b0d89ac6c7e21129454 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 21 Nov 2024 18:43:11 +0800 Subject: [PATCH 114/206] add auto backups --- modules/__init__.py | 4 +++- mytoncore/mytoncore.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/__init__.py b/modules/__init__.py index afb61e14..2b588470 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -58,7 +58,9 @@ class Setting: 'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'), 'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'), 'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'), - 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id') + 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id'), + 'auto_backup': Setting('validator', True, 'Make backup of validator every election'), + 'auto_backup_path': Setting('validator', '/tmp/mytoncore/backups/', 'Path to store backups'), } diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 757660ce..845ff60b 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1455,6 +1455,7 @@ def ElectionEntry(self, args=None): self.local.add_log("ElectionEntry completed. Start work time: " + str(startWorkTime)) self.clear_tmp() + self.make_backup(startWorkTime) #end define @@ -1471,6 +1472,18 @@ def clear_tmp(self): self.local.add_log(f"Removed {count} old files from tmp dir for {int(time.time() - start)} seconds", "info") + def make_backup(self, election_id: str): + if not self.local.db.get("auto_backup"): + return + from mytonctrl.mytonctrl import create_backup + args = [] + name = f"/mytonctrl_backup_elid{election_id}.zip" + if self.local.db.get("auto_backup_path"): + args.append(self.local.db.get("auto_backup_path") + name) + else: + args.append(self.tempDir + "/backups" + name) + create_backup(self.local, self, args + ['-y']) + def GetValidatorKeyByTime(self, startWorkTime, endWorkTime): self.local.add_log("start GetValidatorKeyByTime function", "debug") # Check temp key From 9de0be68b0bdab7eee69c34d2700bb5f322515a9 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 21 Nov 2024 18:53:48 +0800 Subject: [PATCH 115/206] remove cleanup cmd --- mytonctrl/mytonctrl.py | 7 ------ mytonctrl/resources/translate.json | 5 ----- mytonctrl/scripts/cleanup.sh | 35 ------------------------------ 3 files changed, 47 deletions(-) delete mode 100644 mytonctrl/scripts/cleanup.sh diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index de3d7ce8..c185baad 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -140,7 +140,6 @@ def inject_globals(func): module = AlertBotModule(ton, local) module.add_console_commands(console) - console.AddItem("cleanup", inject_globals(cleanup_validator_db), local.translate("cleanup_cmd")) console.AddItem("benchmark", inject_globals(run_benchmark), local.translate("benchmark_cmd")) # console.AddItem("activate_ton_storage_provider", inject_globals(activate_ton_storage_provider), local.translate("activate_ton_storage_provider_cmd")) @@ -403,12 +402,6 @@ def rollback_to_mtc1(local, ton, args): local.exit() #end define -def cleanup_validator_db(ton, args): - cleanup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/cleanup.sh') - run_args = ["bash", cleanup_script_path] - exit_code = run_as_root(run_args) -#end define - def run_benchmark(ton, args): timeout = 200 benchmark_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/benchmark.sh') diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 22915e59..83752783 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -484,11 +484,6 @@ "ru": "Отправить тестовое оповещение через Telegram Bot", "zh_TW": "通過 Telegram Bot 發送測試警報" }, - "cleanup_cmd": { - "en": "Clean node old logs and temp files", - "ru": "Очистить старые логи и временные файлы ноды", - "zh_TW": "清理節點舊日誌和臨時文件" - }, "benchmark_cmd": { "en": "Run benchmark", "ru": "Запустить бенчмарк", diff --git a/mytonctrl/scripts/cleanup.sh b/mytonctrl/scripts/cleanup.sh deleted file mode 100644 index d3d10a2e..00000000 --- a/mytonctrl/scripts/cleanup.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -e - -# Проверить sudo -if [ "$(id -u)" != "0" ]; then - echo "Please run script as root" - exit 1 -fi - -# Цвета -COLOR='\033[92m' -ENDC='\033[0m' - -db_path=/var/ton-work/db - -function get_directory_size { - buff=$(du -sh ${db_path} | awk '{print $1}') - echo ${buff} -} - -echo -e "${COLOR}[1/7]${ENDC} Start node/validator DB cleanup process" -echo -e "${COLOR}[2/7]${ENDC} Stop node/validator" -systemctl stop validator - -echo -e "${COLOR}[3/7]${ENDC} Node/validator DB size before cleanup = $(get_directory_size)" -find /var/ton-work/db -name 'LOG.old*' -exec rm {} + - -echo -e "${COLOR}[4/7]${ENDC} Node/validator DB size after deleting old files = $(get_directory_size)" -rm -r /var/ton-work/db/files/packages/temp.archive.* - -echo -e "${COLOR}[5/7]${ENDC} Node/validator DB size after deleting temporary files = $(get_directory_size)" -echo -e "${COLOR}[6/7]${ENDC} Start node/validator" -systemctl start validator - -echo -e "${COLOR}[7/7]${ENDC} Node/validator DB cleanup process completed" From b61706b5d90591d5e94ac5428bde1333d1cea3b7 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 21 Nov 2024 20:21:10 +0800 Subject: [PATCH 116/206] rm out_of_ser for validators --- mytonctrl/mytonctrl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 4b3dd433..644fe5a9 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -783,7 +783,8 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, print(mytoncoreStatus_text) print(validatorStatus_text) print(validator_out_of_sync_text) - print(validator_out_of_ser_text) + if not ton.using_validator(): # don't need this for validators since they do not serialize states + print(validator_out_of_ser_text) print(dbStatus_text) print(mtcVersion_text) print(validatorVersion_text) From 9861aaf580ffc69b5d92e402e68b4b02a6add987 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 22 Nov 2024 12:46:34 +0800 Subject: [PATCH 117/206] rename dir --- {support => tools}/extract_backup_node_keys.sh | 0 {support => tools}/inject_backup_node_keys.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {support => tools}/extract_backup_node_keys.sh (100%) rename {support => tools}/inject_backup_node_keys.py (100%) diff --git a/support/extract_backup_node_keys.sh b/tools/extract_backup_node_keys.sh similarity index 100% rename from support/extract_backup_node_keys.sh rename to tools/extract_backup_node_keys.sh diff --git a/support/inject_backup_node_keys.py b/tools/inject_backup_node_keys.py similarity index 100% rename from support/inject_backup_node_keys.py rename to tools/inject_backup_node_keys.py From 5a4dd9eac163f025787767fdecaae5f1db0ecedf Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 27 Nov 2024 14:50:08 +0900 Subject: [PATCH 118/206] update node installing scripts --- mytonctrl/scripts/upgrade.sh | 6 +++++- scripts/ton_installer.sh | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mytonctrl/scripts/upgrade.sh b/mytonctrl/scripts/upgrade.sh index 81a9fd0b..c3c4c52b 100644 --- a/mytonctrl/scripts/upgrade.sh +++ b/mytonctrl/scripts/upgrade.sh @@ -29,7 +29,7 @@ COLOR='\033[92m' ENDC='\033[0m' # Установить дополнительные зависимости -apt-get install -y libsecp256k1-dev libsodium-dev ninja-build fio rocksdb-tools liblz4-dev libjemalloc-dev +apt-get install -y libsecp256k1-dev libsodium-dev ninja-build fio rocksdb-tools liblz4-dev libjemalloc-dev automake libtool # bugfix if the files are in the wrong place wget "https://ton-blockchain.github.io/global.config.json" -O global.config.json @@ -67,6 +67,10 @@ ls -A1 | xargs rm -rf echo "https://github.com/${author}/${repo}.git -> ${branch}" git clone --recursive https://github.com/${author}/${repo}.git . git checkout ${branch} + +git submodule sync --recursive +git submodule update + export CC=/usr/bin/clang export CXX=/usr/bin/clang++ export CCACHE_DISABLE=1 diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index 6d4ad84a..6454fd2a 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -55,7 +55,7 @@ if [ "$OSTYPE" == "linux-gnu" ]; then elif [ -f /etc/debian_version ]; then echo "Ubuntu/Debian Linux detected." apt-get update - apt-get install -y build-essential curl git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev + apt-get install -y build-essential curl git cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip libsecp256k1-dev libsodium-dev liblz4-dev libjemalloc-dev automake libtool # Install ninja apt-get install -y ninja-build @@ -118,6 +118,9 @@ if [ "${ton_node_version}" != "master" ]; then cd ../ fi +git submodule sync --recursive +git submodule update + git config --global --add safe.directory $SOURCES_DIR/ton # Подготавливаем папки для компиляции From 2977185aad5b37793e163b9ef3231a717e2be21c Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 27 Nov 2024 14:42:25 +0900 Subject: [PATCH 119/206] delete old auto backups --- mytoncore/mytoncore.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 845ff60b..00cde9df 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1459,18 +1459,19 @@ def ElectionEntry(self, args=None): #end define - def clear_tmp(self): + def clear_dir(self, dir_name): start = time.time() count = 0 week_ago = 60 * 60 * 24 * 7 - dir = self.tempDir - for f in os.listdir(dir): - ts = os.path.getmtime(os.path.join(dir, f)) + for f in os.listdir(dir_name): + ts = os.path.getmtime(os.path.join(dir_name, f)) if ts < time.time() - week_ago: count += 1 - os.remove(os.path.join(dir, f)) + os.remove(os.path.join(dir_name, f)) + self.local.add_log(f"Removed {count} old files from {dir_name} directory for {int(time.time() - start)} seconds", "info") - self.local.add_log(f"Removed {count} old files from tmp dir for {int(time.time() - start)} seconds", "info") + def clear_tmp(self): + self.clear_dir(self.tempDir) def make_backup(self, election_id: str): if not self.local.db.get("auto_backup"): @@ -1478,10 +1479,12 @@ def make_backup(self, election_id: str): from mytonctrl.mytonctrl import create_backup args = [] name = f"/mytonctrl_backup_elid{election_id}.zip" + backups_dir = self.tempDir + "/auto_backups" if self.local.db.get("auto_backup_path"): - args.append(self.local.db.get("auto_backup_path") + name) - else: - args.append(self.tempDir + "/backups" + name) + backups_dir = self.local.db.get("auto_backup_path") + os.makedirs(self.tempDir + "/auto_backups", exist_ok=True) + args.append(backups_dir + name) + self.clear_dir(backups_dir) create_backup(self.local, self, args + ['-y']) def GetValidatorKeyByTime(self, startWorkTime, endWorkTime): From 16cf7583f56f9ab69e1ddb5780b4fb42495a79f4 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 28 Nov 2024 14:13:57 +0900 Subject: [PATCH 120/206] support multiple node args with same name --- mytoninstaller/mytoninstaller.py | 6 +++-- mytoninstaller/node_args.py | 29 ++++++++++++++++++--- mytoninstaller/scripts/set_node_argument.py | 7 +++-- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index 270afe97..97e5ce4e 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -133,13 +133,15 @@ def Status(local, args): node_args = get_node_args() color_print("{cyan}===[ Node arguments ]==={endc}") for key, value in node_args.items(): - print(f"{key}: {value}") + for v in value: + print(f"{key}: {v}") #end define def set_node_argument(local, args): if len(args) < 1: - color_print("{red}Bad args. Usage:{endc} set_node_argument [arg-value] [-d (to delete)]") + color_print("{red}Bad args. Usage:{endc} set_node_argument [arg-value] [-d (to delete)].\n" + "Examples: 'set_node_argument --archive-ttl 86400' or 'set_node_argument --archive-ttl -d' or 'set_node_argument -M' or 'set_node_argument --add-shard 0:2000000000000000 0:a000000000000000'") return arg_name = args[0] args = [arg_name, args[1] if len(args) > 1 else ""] diff --git a/mytoninstaller/node_args.py b/mytoninstaller/node_args.py index af7b8380..e57d925b 100644 --- a/mytoninstaller/node_args.py +++ b/mytoninstaller/node_args.py @@ -15,21 +15,42 @@ def get_node_start_command(): #end define +""" def get_node_args(command: str = None): if command is None: command = get_node_start_command() - result = {} + result = [] key = '' for c in command.split(' ')[1:]: if c.startswith('--') or c.startswith('-'): if key: - result[key] = '' + result.append([key, '']) key = c elif key: - result[key] = c + result.append([key, c]) key = '' if key: - result[key] = '' + result.append([key, '']) + return result +#end define +""" + + +def get_node_args(command: str = None): + if command is None: + command = get_node_start_command() + result = {} # {key: [value1, value2]} + key = '' + for c in command.split(' ')[1:]: + if c.startswith('--') or c.startswith('-'): + if key: + result[key] = result.get(key, []) + [''] + key = c + elif key: + result[key] = result.get(key, []) + [c] + key = '' + if key: + result[key] = result.get(key, []) + [''] return result #end define diff --git a/mytoninstaller/scripts/set_node_argument.py b/mytoninstaller/scripts/set_node_argument.py index 19fe67a5..cd241bea 100644 --- a/mytoninstaller/scripts/set_node_argument.py +++ b/mytoninstaller/scripts/set_node_argument.py @@ -20,8 +20,11 @@ def set_node_arg(arg_name: str, arg_value: str = ''): if arg_value == '-d': args.pop(arg_name, None) else: - args[arg_name] = arg_value - new_command = command.split(' ')[0] + ' ' + ' '.join([f'{k} {v}' for k, v in args.items()]) + if ' ' in arg_value: + args[arg_name] = arg_value.split() + else: + args[arg_name] = [arg_value] + new_command = command.split(' ')[0] + ' ' + ' '.join([f'{k} {v}' for k, vs in args.items() for v in vs]) new_service = service.replace(command, new_command) with open('/etc/systemd/system/validator.service', 'w') as f: f.write(new_service) From e0d87d1b30810608fdb007afed84fd4bcce54e43 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 28 Nov 2024 14:16:49 +0900 Subject: [PATCH 121/206] dont remove state-ttl for vals on updates --- mytonctrl/mytonctrl.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index de3d7ce8..25e340fc 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -369,14 +369,6 @@ def Upgrade(ton, args): upgrade_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/upgrade.sh') runArgs = ["bash", upgrade_script_path, "-a", author, "-r", repo, "-b", branch] exitCode = run_as_root(runArgs) - if ton.using_validator(): - try: - from mytoninstaller.mytoninstaller import set_node_argument, get_node_args - node_args = get_node_args() - if node_args.get('--state-ttl') == '604800': - set_node_argument(ton.local, ["--state-ttl", "-d"]) - except Exception as e: - color_print(f"{{red}}Failed to set node argument: {e} {{endc}}") if exitCode == 0: text = "Upgrade - {green}OK{endc}" else: From c00f8c37226a6738cd5cd44357bcb300ad27e5ee Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 28 Nov 2024 19:35:21 +0900 Subject: [PATCH 122/206] add ton_dir option to create_backup.sh --- mytonctrl/scripts/create_backup.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mytonctrl/scripts/create_backup.sh b/mytonctrl/scripts/create_backup.sh index 0c88788a..2b039fe0 100644 --- a/mytonctrl/scripts/create_backup.sh +++ b/mytonctrl/scripts/create_backup.sh @@ -1,12 +1,14 @@ dest="mytonctrl_backup_$(hostname)_$(date +%s).tar.gz" mtc_dir="$HOME/.local/share/mytoncore" user=$(logname) +ton_dir="/var/ton-work" # Get arguments -while getopts d:m: flag +while getopts d:m:t: flag do case "${flag}" in d) dest=${OPTARG};; m) mtc_dir=${OPTARG};; + t) ton_dir=${OPTARG};; *) echo "Flag -${flag} is not recognized. Aborting" exit 1 ;; @@ -26,9 +28,9 @@ rm -rf $tmp_dir mkdir $tmp_dir mkdir $tmp_dir/db -cp /var/ton-work/db/config.json ${tmp_dir}/db -cp -r /var/ton-work/db/keyring ${tmp_dir}/db -cp -r /var/ton-work/keys ${tmp_dir} +cp $ton_dir/db/config.json ${tmp_dir}/db +cp -r $ton_dir/db/keyring ${tmp_dir}/db +cp -r $ton_dir/keys ${tmp_dir} cp -r $mtc_dir $tmp_dir python3 -c "import json;f=open('${tmp_dir}/db/config.json');json.load(f);f.close()" || exit 1 # Check if config.json is copied correctly From 118dd69b0f27ecf40996570265eb6a56c3e82556 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 28 Nov 2024 14:13:57 +0900 Subject: [PATCH 123/206] support multiple node args with same name --- mytoninstaller/mytoninstaller.py | 8 +++++--- mytoninstaller/node_args.py | 8 ++++---- mytoninstaller/scripts/set_node_argument.py | 7 +++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index 270afe97..24e964ea 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -133,16 +133,18 @@ def Status(local, args): node_args = get_node_args() color_print("{cyan}===[ Node arguments ]==={endc}") for key, value in node_args.items(): - print(f"{key}: {value}") + for v in value: + print(f"{key}: {v}") #end define def set_node_argument(local, args): if len(args) < 1: - color_print("{red}Bad args. Usage:{endc} set_node_argument [arg-value] [-d (to delete)]") + color_print("{red}Bad args. Usage:{endc} set_node_argument [arg-value] [-d (to delete)].\n" + "Examples: 'set_node_argument --archive-ttl 86400' or 'set_node_argument --archive-ttl -d' or 'set_node_argument -M' or 'set_node_argument --add-shard 0:2000000000000000 0:a000000000000000'") return arg_name = args[0] - args = [arg_name, args[1] if len(args) > 1 else ""] + args = [arg_name, " ".join(args[1:])] script_path = pkg_resources.resource_filename('mytoninstaller.scripts', 'set_node_argument.py') run_as_root(['python3', script_path] + args) color_print("set_node_argument - {green}OK{endc}") diff --git a/mytoninstaller/node_args.py b/mytoninstaller/node_args.py index af7b8380..0c5fe3d0 100644 --- a/mytoninstaller/node_args.py +++ b/mytoninstaller/node_args.py @@ -18,18 +18,18 @@ def get_node_start_command(): def get_node_args(command: str = None): if command is None: command = get_node_start_command() - result = {} + result = {} # {key: [value1, value2]} key = '' for c in command.split(' ')[1:]: if c.startswith('--') or c.startswith('-'): if key: - result[key] = '' + result[key] = result.get(key, []) + [''] key = c elif key: - result[key] = c + result[key] = result.get(key, []) + [c] key = '' if key: - result[key] = '' + result[key] = result.get(key, []) + [''] return result #end define diff --git a/mytoninstaller/scripts/set_node_argument.py b/mytoninstaller/scripts/set_node_argument.py index 19fe67a5..cd241bea 100644 --- a/mytoninstaller/scripts/set_node_argument.py +++ b/mytoninstaller/scripts/set_node_argument.py @@ -20,8 +20,11 @@ def set_node_arg(arg_name: str, arg_value: str = ''): if arg_value == '-d': args.pop(arg_name, None) else: - args[arg_name] = arg_value - new_command = command.split(' ')[0] + ' ' + ' '.join([f'{k} {v}' for k, v in args.items()]) + if ' ' in arg_value: + args[arg_name] = arg_value.split() + else: + args[arg_name] = [arg_value] + new_command = command.split(' ')[0] + ' ' + ' '.join([f'{k} {v}' for k, vs in args.items() for v in vs]) new_service = service.replace(command, new_command) with open('/etc/systemd/system/validator.service', 'w') as f: f.write(new_service) From 37593eb4b123a3f4e563c5e08f7bf498c584f0e6 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 28 Nov 2024 14:16:49 +0900 Subject: [PATCH 124/206] dont remove state-ttl for vals on updates --- mytonctrl/mytonctrl.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index de3d7ce8..25e340fc 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -369,14 +369,6 @@ def Upgrade(ton, args): upgrade_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/upgrade.sh') runArgs = ["bash", upgrade_script_path, "-a", author, "-r", repo, "-b", branch] exitCode = run_as_root(runArgs) - if ton.using_validator(): - try: - from mytoninstaller.mytoninstaller import set_node_argument, get_node_args - node_args = get_node_args() - if node_args.get('--state-ttl') == '604800': - set_node_argument(ton.local, ["--state-ttl", "-d"]) - except Exception as e: - color_print(f"{{red}}Failed to set node argument: {e} {{endc}}") if exitCode == 0: text = "Upgrade - {green}OK{endc}" else: From 647f34e18f84310e6ff427b81c2c77bb9b07bf19 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 2 Dec 2024 20:14:46 +0900 Subject: [PATCH 125/206] add add-shard to installer --- mytoninstaller/settings.py | 7 +++++++ scripts/install.py | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index cba219ea..b9ce9940 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -63,6 +63,13 @@ def FirstNodeSettings(local): # Прописать автозагрузку cpus = psutil.cpu_count() - 1 cmd = f"{validatorAppPath} --threads {cpus} --daemonize --global-config {globalConfigPath} --db {ton_db_dir} --logname {tonLogPath} --archive-ttl {archive_ttl} --verbosity 1" + + if os.getenv('ADD_SHARD'): + add_shard = os.getenv('ADD_SHARD') + cmd += f' -M' + for shard in add_shard.split(): + cmd += f' --add-shard {shard}' + add2systemd(name="validator", user=vuser, start=cmd) # post="/usr/bin/python3 /usr/src/mytonctrl/mytoncore.py -e \"validator down\"" # Получить внешний ip адрес diff --git a/scripts/install.py b/scripts/install.py index aa43c0ac..bfb569b4 100644 --- a/scripts/install.py +++ b/scripts/install.py @@ -38,6 +38,13 @@ def run_cli(): "dump", message="Do you want to download blockchain's dump? " "This reduces synchronization time but requires to download a large file", + ), + inquirer.Text( + "add-shard", + message="Set shards node will sync. Skip to sync all shards. " + "Format: :. Divide multiple shards with space. " + "Example: `0:2000000000000000 0:6000000000000000`", + validate=lambda _, x: not x or all([":" in i for i in x.split()]) ) ] @@ -51,6 +58,7 @@ def parse_args(answers: dict): network = answers["network"].lower() config = answers["config"] archive_ttl = answers["archive-ttl"] + add_shard = answers["add-shard"] validator_mode = answers["validator-mode"] dump = answers["dump"] @@ -61,6 +69,8 @@ def parse_args(answers: dict): if archive_ttl: os.putenv('ARCHIVE_TTL', archive_ttl) # set env variable + if add_shard: + os.putenv('ADD_SHARD', add_shard) if validator_mode and validator_mode not in ('Skip', 'Validator wallet'): if validator_mode == 'Nominator pool': From 8b916ad74dd7d62eb719806d7a5e846366a05f9b Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 5 Dec 2024 13:59:33 +0900 Subject: [PATCH 126/206] add node ip to alerting --- modules/alert_bot.py | 3 +++ mytoncore/mytoncore.py | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index aea45732..d84d5e92 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -83,6 +83,7 @@ def __init__(self, ton, local, *args, **kwargs): self.validator_module = None self.inited = False self.hostname = None + self.ip = None self.token = self.ton.local.db.get("BotToken") self.chat_id = self.ton.local.db.get("ChatId") self.last_db_check = None @@ -113,6 +114,7 @@ def send_alert(self, alert_name: str, *args, **kwargs): ❗️ MyTonCtrl Alert {alert_name} ❗️ Hostname: {self.hostname} +Node IP: {self.ip} Time: {time_} ({int(time.time())}) Severity: {alert.severity} @@ -138,6 +140,7 @@ def init(self): from modules.validator import ValidatorModule self.validator_module = ValidatorModule(self.ton, self.local) self.hostname = get_hostname() + self.ip = self.ton.get_validator_engine_ip() self.set_global_vars() self.inited = True diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 76cf58cd..7523a51a 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -30,7 +30,7 @@ get_timestamp, timestamp2datetime, dec2hex, - Dict + Dict, int2ip ) @@ -3791,6 +3791,13 @@ def GetNetworkName(self): return "unknown" #end define + def get_validator_engine_ip(self): + try: + config = self.GetValidatorConfig() + return int2ip(config['addrs'][0]['ip']) + except: + return None + def GetFunctionBuffer(self, name, timeout=10): timestamp = get_timestamp() buff = self.local.buffer.get(name) From 6b43862e6d97d3663ba8d09ec4d096af39ac45d1 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 5 Dec 2024 20:20:24 +0900 Subject: [PATCH 127/206] do not send balance alert when out of sync --- modules/alert_bot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index e55fcc61..2ebf4034 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -209,6 +209,9 @@ def check_db_usage(self): def check_validator_wallet_balance(self): if not self.ton.using_validator(): return + validator_status = self.ton.GetValidatorStatus() + if not validator_status.is_working or validator_status.out_of_sync >= 20: + return validator_wallet = self.ton.GetValidatorWallet() validator_account = self.ton.GetAccount(validator_wallet.addrB64) if validator_account.balance < 10: From c383356a962d9c40b0c5d440267932200a225957 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 29 Nov 2024 13:40:29 +0900 Subject: [PATCH 128/206] move backups to separate module --- modules/__init__.py | 4 +-- modules/backups.py | 58 ++++++++++++++++++++++++++++++++++++++++++ mytoncore/mytoncore.py | 11 +++++--- mytonctrl/mytonctrl.py | 53 +++----------------------------------- 4 files changed, 72 insertions(+), 54 deletions(-) create mode 100644 modules/backups.py diff --git a/modules/__init__.py b/modules/__init__.py index 2b588470..3d09bb9e 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -59,8 +59,8 @@ class Setting: 'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'), 'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'), 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id'), - 'auto_backup': Setting('validator', True, 'Make backup of validator every election'), - 'auto_backup_path': Setting('validator', '/tmp/mytoncore/backups/', 'Path to store backups'), + 'auto_backup': Setting('validator', None, 'Make validator backup every election'), + 'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'), } diff --git a/modules/backups.py b/modules/backups.py new file mode 100644 index 00000000..31930ce5 --- /dev/null +++ b/modules/backups.py @@ -0,0 +1,58 @@ +import pkg_resources + +from modules.module import MtcModule +from mypylib.mypylib import color_print, ip2int, run_as_root +from mytoninstaller.config import get_own_ip + + +class BackupModule(MtcModule): + + def create_backup(self, args): + if len(args) > 2: + color_print("{red}Bad args. Usage:{endc} create_backup [path_to_archive] [-y]") + return + if '-y' not in args: + res = input(f'Mytoncore service will be stopped for few seconds while backup is created, Proceed [y/n]?') + if res.lower() != 'y': + print('aborted.') + return + else: + args.pop(args.index('-y')) + command_args = ["-m", self.ton.local.buffer.my_work_dir] + if len(args) == 1: + command_args += ["-d", args[0]] + backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh') + if run_as_root(["bash", backup_script_path] + command_args) == 0: + color_print("create_backup - {green}OK{endc}") + else: + color_print("create_backup - {red}Error{endc}") + # end define + + def restore_backup(self, args): + if len(args) == 0 or len(args) > 2: + color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") + return + if '-y' not in args: + res = input( + f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]') + if res.lower() != 'y': + print('aborted.') + return + else: + args.pop(args.index('-y')) + print('Before proceeding, mtc will create a backup of current configuration.') + self.create_backup(['-y']) + ip = str(ip2int(get_own_ip())) + command_args = ["-m", self.ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip] + + restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh') + if run_as_root(["bash", restore_script_path] + command_args) == 0: + color_print("restore_backup - {green}OK{endc}") + self.local.exit() + else: + color_print("restore_backup - {red}Error{endc}") + # end define + + def add_console_commands(self, console): + console.AddItem("create_backup", self.create_backup, self.local.translate("create_backup_cmd")) + console.AddItem("restore_backup", self.restore_backup, self.local.translate("restore_backup_cmd")) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 00cde9df..afc1829f 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1476,16 +1476,21 @@ def clear_tmp(self): def make_backup(self, election_id: str): if not self.local.db.get("auto_backup"): return - from mytonctrl.mytonctrl import create_backup + from modules.backups import BackupModule + module = BackupModule(self, self.local) args = [] name = f"/mytonctrl_backup_elid{election_id}.zip" backups_dir = self.tempDir + "/auto_backups" if self.local.db.get("auto_backup_path"): backups_dir = self.local.db.get("auto_backup_path") - os.makedirs(self.tempDir + "/auto_backups", exist_ok=True) + os.makedirs(backups_dir, exist_ok=True) args.append(backups_dir + name) self.clear_dir(backups_dir) - create_backup(self.local, self, args + ['-y']) + exit_code = module.create_backup(args + ['-y']) + if exit_code != 0: + self.local.add_log(f"Backup failed with exit code {exit_code}", "error") + else: + self.local.add_log(f"Backup created successfully", "info") def GetValidatorKeyByTime(self, startWorkTime, endWorkTime): self.local.add_log("start GetValidatorKeyByTime function", "debug") diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index de3d7ce8..e3d30777 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -85,8 +85,6 @@ def inject_globals(func): console.AddItem("get", inject_globals(GetSettings), local.translate("get_cmd")) console.AddItem("set", inject_globals(SetSettings), local.translate("set_cmd")) console.AddItem("rollback", inject_globals(rollback_to_mtc1), local.translate("rollback_cmd")) - console.AddItem("create_backup", inject_globals(create_backup), local.translate("create_backup_cmd")) - console.AddItem("restore_backup", inject_globals(restore_backup), local.translate("restore_backup_cmd")) #console.AddItem("xrestart", inject_globals(Xrestart), local.translate("xrestart_cmd")) #console.AddItem("xlist", inject_globals(Xlist), local.translate("xlist_cmd")) @@ -94,6 +92,10 @@ def inject_globals(func): #console.AddItem("ssoc", inject_globals(SignShardOverlayCert), local.translate("ssoc_cmd")) #console.AddItem("isoc", inject_globals(ImportShardOverlayCert), local.translate("isoc_cmd")) + from modules.backups import BackupModule + module = BackupModule(ton, local) + module.add_console_commands(console) + from modules.custom_overlays import CustomOverlayModule module = CustomOverlayModule(ton, local) module.add_console_commands(console) @@ -939,53 +941,6 @@ def disable_mode(local, ton, args): #end define -def create_backup(local, ton, args): - if len(args) > 2: - color_print("{red}Bad args. Usage:{endc} create_backup [path_to_archive] [-y]") - return - if '-y' not in args: - res = input(f'Mytoncore service will be stopped for few seconds while backup is created, Proceed [y/n]?') - if res.lower() != 'y': - print('aborted.') - return - else: - args.pop(args.index('-y')) - command_args = ["-m", ton.local.buffer.my_work_dir] - if len(args) == 1: - command_args += ["-d", args[0]] - backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh') - if run_as_root(["bash", backup_script_path] + command_args) == 0: - color_print("create_backup - {green}OK{endc}") - else: - color_print("create_backup - {red}Error{endc}") -#end define - - -def restore_backup(local, ton, args): - if len(args) == 0 or len(args) > 2: - color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") - return - if '-y' not in args: - res = input(f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]') - if res.lower() != 'y': - print('aborted.') - return - else: - args.pop(args.index('-y')) - print('Before proceeding, mtc will create a backup of current configuration.') - create_backup(local, ton, ['-y']) - ip = str(ip2int(get_own_ip())) - command_args = ["-m", ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip] - - restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh') - if run_as_root(["bash", restore_script_path] + command_args) == 0: - color_print("restore_backup - {green}OK{endc}") - local.exit() - else: - color_print("restore_backup - {red}Error{endc}") -#end define - - def Xrestart(inputArgs): if len(inputArgs) < 2: color_print("{red}Bad args. Usage:{endc} xrestart ") From 6a9a2c5692bc5511b337d669969205b66fad7dfe Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 2 Dec 2024 18:39:39 +0900 Subject: [PATCH 129/206] get keys and node config for backups from v-console --- modules/backups.py | 33 +++++++++++++++++++++++++----- mytonctrl/scripts/create_backup.sh | 6 ++++-- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/modules/backups.py b/modules/backups.py index 31930ce5..33de5867 100644 --- a/modules/backups.py +++ b/modules/backups.py @@ -1,15 +1,34 @@ +import os +import subprocess +import time + import pkg_resources from modules.module import MtcModule -from mypylib.mypylib import color_print, ip2int, run_as_root +from mypylib.mypylib import color_print, ip2int, run_as_root, parse from mytoninstaller.config import get_own_ip class BackupModule(MtcModule): + def create_keyring(self, dir_name): + keyring_dir = dir_name + '/keyring' + self.ton.validatorConsole.Run(f'exportallprivatekeys {keyring_dir}') + + def create_tmp_ton_dir(self): + result = self.ton.validatorConsole.Run("getconfig") + text = parse(result, "---------", "--------") + dir_name = self.ton.tempDir + f'/ton_backup_{int(time.time() * 1000)}' + dir_name_db = dir_name + '/db' + os.makedirs(dir_name_db) + with open(dir_name_db + '/config.json', 'w') as f: + f.write(text) + self.create_keyring(dir_name_db) + return dir_name + def create_backup(self, args): if len(args) > 2: - color_print("{red}Bad args. Usage:{endc} create_backup [path_to_archive] [-y]") + color_print("{red}Bad args. Usage:{endc} create_backup [filename] [-y]") return if '-y' not in args: res = input(f'Mytoncore service will be stopped for few seconds while backup is created, Proceed [y/n]?') @@ -18,19 +37,23 @@ def create_backup(self, args): return else: args.pop(args.index('-y')) - command_args = ["-m", self.ton.local.buffer.my_work_dir] + dir_ = self.create_tmp_ton_dir() + command_args = ["-m", self.ton.local.buffer.my_work_dir, "-t", dir_] if len(args) == 1: command_args += ["-d", args[0]] backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh') - if run_as_root(["bash", backup_script_path] + command_args) == 0: + process = subprocess.run(["bash", backup_script_path] + command_args, timeout=5) + + if process.returncode == 0: color_print("create_backup - {green}OK{endc}") else: color_print("create_backup - {red}Error{endc}") + return process.returncode # end define def restore_backup(self, args): if len(args) == 0 or len(args) > 2: - color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") + color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") return if '-y' not in args: res = input( diff --git a/mytonctrl/scripts/create_backup.sh b/mytonctrl/scripts/create_backup.sh index 2b039fe0..1df4e23d 100644 --- a/mytonctrl/scripts/create_backup.sh +++ b/mytonctrl/scripts/create_backup.sh @@ -2,13 +2,15 @@ dest="mytonctrl_backup_$(hostname)_$(date +%s).tar.gz" mtc_dir="$HOME/.local/share/mytoncore" user=$(logname) ton_dir="/var/ton-work" +keys_dir="/var/ton-work/keys" # Get arguments -while getopts d:m:t: flag +while getopts d:m:t:k: flag do case "${flag}" in d) dest=${OPTARG};; m) mtc_dir=${OPTARG};; t) ton_dir=${OPTARG};; + k) keys_dir=${OPTARG};; *) echo "Flag -${flag} is not recognized. Aborting" exit 1 ;; @@ -30,7 +32,7 @@ mkdir $tmp_dir/db cp $ton_dir/db/config.json ${tmp_dir}/db cp -r $ton_dir/db/keyring ${tmp_dir}/db -cp -r $ton_dir/keys ${tmp_dir} +cp -r $keys_dir ${tmp_dir} cp -r $mtc_dir $tmp_dir python3 -c "import json;f=open('${tmp_dir}/db/config.json');json.load(f);f.close()" || exit 1 # Check if config.json is copied correctly From 4f4873271d7793d279a13169bd4e3e204887bf3d Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 4 Dec 2024 21:23:30 +0900 Subject: [PATCH 130/206] remove mytoncore restart from backup, remove -y flag --- modules/backups.py | 13 +++---------- mytoncore/mytoncore.py | 2 +- mytonctrl/scripts/create_backup.sh | 16 +++++----------- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/modules/backups.py b/modules/backups.py index 33de5867..da4a9953 100644 --- a/modules/backups.py +++ b/modules/backups.py @@ -27,16 +27,9 @@ def create_tmp_ton_dir(self): return dir_name def create_backup(self, args): - if len(args) > 2: - color_print("{red}Bad args. Usage:{endc} create_backup [filename] [-y]") + if len(args) > 1: + color_print("{red}Bad args. Usage:{endc} create_backup [filename]") return - if '-y' not in args: - res = input(f'Mytoncore service will be stopped for few seconds while backup is created, Proceed [y/n]?') - if res.lower() != 'y': - print('aborted.') - return - else: - args.pop(args.index('-y')) dir_ = self.create_tmp_ton_dir() command_args = ["-m", self.ton.local.buffer.my_work_dir, "-t", dir_] if len(args) == 1: @@ -64,7 +57,7 @@ def restore_backup(self, args): else: args.pop(args.index('-y')) print('Before proceeding, mtc will create a backup of current configuration.') - self.create_backup(['-y']) + self.create_backup([]) ip = str(ip2int(get_own_ip())) command_args = ["-m", self.ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip] diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index afc1829f..27cfe985 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1486,7 +1486,7 @@ def make_backup(self, election_id: str): os.makedirs(backups_dir, exist_ok=True) args.append(backups_dir + name) self.clear_dir(backups_dir) - exit_code = module.create_backup(args + ['-y']) + exit_code = module.create_backup(args) if exit_code != 0: self.local.add_log(f"Backup failed with exit code {exit_code}", "error") else: diff --git a/mytonctrl/scripts/create_backup.sh b/mytonctrl/scripts/create_backup.sh index 1df4e23d..1134f2bf 100644 --- a/mytonctrl/scripts/create_backup.sh +++ b/mytonctrl/scripts/create_backup.sh @@ -20,12 +20,7 @@ done COLOR='\033[92m' ENDC='\033[0m' -systemctl stop mytoncore - -echo -e "${COLOR}[1/4]${ENDC} Stopped mytoncore service" - - -tmp_dir="/tmp/mytoncore/backup" +tmp_dir="/tmp/mytoncore/backupv2" rm -rf $tmp_dir mkdir $tmp_dir mkdir $tmp_dir/db @@ -36,16 +31,15 @@ cp -r $keys_dir ${tmp_dir} cp -r $mtc_dir $tmp_dir python3 -c "import json;f=open('${tmp_dir}/db/config.json');json.load(f);f.close()" || exit 1 # Check if config.json is copied correctly +python3 -c "import json;f=open('${tmp_dir}/mytoncore/mytoncore.db');json.load(f);f.close()" || exit 2 # Check if mytoncore.db is copied correctly -echo -e "${COLOR}[2/4]${ENDC} Copied files to ${tmp_dir}" - -systemctl start mytoncore +echo -e "${COLOR}[1/3]${ENDC} Copied files to ${tmp_dir}" -echo -e "${COLOR}[3/4]${ENDC} Started mytoncore service" +echo -e "${COLOR}[2/3]${ENDC} Started mytoncore service" tar -zcf $dest -C $tmp_dir . chown $user:$user $dest -echo -e "${COLOR}[4/4]${ENDC} Backup successfully created in ${dest}!" +echo -e "${COLOR}[3/3]${ENDC} Backup successfully created in ${dest}!" echo -e "If you wish to use archive package to migrate node to different machine please make sure to stop validator and mytoncore on donor (this) host prior to migration." From ba1e1c3ec6f7f27df4984c668ce81015e0d166bc Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 4 Dec 2024 21:30:12 +0900 Subject: [PATCH 131/206] retry auto backup if error --- mytoncore/mytoncore.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 27cfe985..51b1e87e 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1489,7 +1489,11 @@ def make_backup(self, election_id: str): exit_code = module.create_backup(args) if exit_code != 0: self.local.add_log(f"Backup failed with exit code {exit_code}", "error") - else: + # try one more time + exit_code = module.create_backup(args) + if exit_code != 0: + self.local.add_log(f"Backup failed with exit code {exit_code}", "error") + if exit_code == 0: self.local.add_log(f"Backup created successfully", "info") def GetValidatorKeyByTime(self, startWorkTime, endWorkTime): From 1de94e6d1b32f95191fcf3a5c19d324844501177 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 5 Dec 2024 14:32:48 +0900 Subject: [PATCH 132/206] rm tmp backup dir --- modules/backups.py | 6 ++++-- mytoncore/mytoncore.py | 3 ++- mytonctrl/scripts/create_backup.sh | 9 +++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/backups.py b/modules/backups.py index da4a9953..8ddffae2 100644 --- a/modules/backups.py +++ b/modules/backups.py @@ -1,4 +1,5 @@ import os +import shutil import subprocess import time @@ -30,8 +31,8 @@ def create_backup(self, args): if len(args) > 1: color_print("{red}Bad args. Usage:{endc} create_backup [filename]") return - dir_ = self.create_tmp_ton_dir() - command_args = ["-m", self.ton.local.buffer.my_work_dir, "-t", dir_] + tmp_dir = self.create_tmp_ton_dir() + command_args = ["-m", self.ton.local.buffer.my_work_dir, "-t", tmp_dir] if len(args) == 1: command_args += ["-d", args[0]] backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh') @@ -41,6 +42,7 @@ def create_backup(self, args): color_print("create_backup - {green}OK{endc}") else: color_print("create_backup - {red}Error{endc}") + shutil.rmtree(tmp_dir) return process.returncode # end define diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 51b1e87e..3bde9ae6 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1467,7 +1467,8 @@ def clear_dir(self, dir_name): ts = os.path.getmtime(os.path.join(dir_name, f)) if ts < time.time() - week_ago: count += 1 - os.remove(os.path.join(dir_name, f)) + if os.path.isfile(os.path.join(dir_name, f)): + os.remove(os.path.join(dir_name, f)) self.local.add_log(f"Removed {count} old files from {dir_name} directory for {int(time.time() - start)} seconds", "info") def clear_tmp(self): diff --git a/mytonctrl/scripts/create_backup.sh b/mytonctrl/scripts/create_backup.sh index 1134f2bf..dbcfe5b8 100644 --- a/mytonctrl/scripts/create_backup.sh +++ b/mytonctrl/scripts/create_backup.sh @@ -33,13 +33,14 @@ cp -r $mtc_dir $tmp_dir python3 -c "import json;f=open('${tmp_dir}/db/config.json');json.load(f);f.close()" || exit 1 # Check if config.json is copied correctly python3 -c "import json;f=open('${tmp_dir}/mytoncore/mytoncore.db');json.load(f);f.close()" || exit 2 # Check if mytoncore.db is copied correctly -echo -e "${COLOR}[1/3]${ENDC} Copied files to ${tmp_dir}" - -echo -e "${COLOR}[2/3]${ENDC} Started mytoncore service" +echo -e "${COLOR}[1/2]${ENDC} Copied files to ${tmp_dir}" tar -zcf $dest -C $tmp_dir . chown $user:$user $dest -echo -e "${COLOR}[3/3]${ENDC} Backup successfully created in ${dest}!" +echo -e "${COLOR}[2/2]${ENDC} Backup successfully created in ${dest}!" + +rm -rf $tmp_dir + echo -e "If you wish to use archive package to migrate node to different machine please make sure to stop validator and mytoncore on donor (this) host prior to migration." From 5f8f8391007ea161a98826dddabf262b55a4adc1 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 10 Dec 2024 10:30:34 +0900 Subject: [PATCH 133/206] Revert "rm out_of_ser for validators" This reverts commit b61706b5d90591d5e94ac5428bde1333d1cea3b7. --- mytonctrl/mytonctrl.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 644fe5a9..4b3dd433 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -783,8 +783,7 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, print(mytoncoreStatus_text) print(validatorStatus_text) print(validator_out_of_sync_text) - if not ton.using_validator(): # don't need this for validators since they do not serialize states - print(validator_out_of_ser_text) + print(validator_out_of_ser_text) print(dbStatus_text) print(mtcVersion_text) print(validatorVersion_text) From d82c371e6ae9e5bde04d82ae645519efc8c37dd8 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 11 Dec 2024 13:57:47 +0900 Subject: [PATCH 134/206] do not abort restore_backup if could not create backup --- modules/backups.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/backups.py b/modules/backups.py index 8ddffae2..7ed70954 100644 --- a/modules/backups.py +++ b/modules/backups.py @@ -59,7 +59,11 @@ def restore_backup(self, args): else: args.pop(args.index('-y')) print('Before proceeding, mtc will create a backup of current configuration.') - self.create_backup([]) + try: + self.create_backup([]) + except: + color_print("{red}Could not create backup{endc}") + ip = str(ip2int(get_own_ip())) command_args = ["-m", self.ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip] From 77d550a602ab9ca2a6a0e28b54e97f926e214f4c Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 11 Dec 2024 22:38:51 +0900 Subject: [PATCH 135/206] add only-mtc and backup flags to installation --- mytoninstaller/config.py | 2 ++ mytoninstaller/mytoninstaller.py | 19 ++++++++++++++++++- mytoninstaller/settings.py | 20 ++++++++++++++++++++ scripts/install.sh | 8 ++++++-- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/mytoninstaller/config.py b/mytoninstaller/config.py index 83039853..1be86b79 100644 --- a/mytoninstaller/config.py +++ b/mytoninstaller/config.py @@ -44,6 +44,8 @@ def backup_config(local, config_path): def BackupVconfig(local): + if local.buffer.only_mtc: + return local.add_log("Backup validator config file 'config.json' to 'config.json.backup'", "debug") vconfig_path = local.buffer.vconfig_path backupPath = vconfig_path + ".backup" diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index 270afe97..2d54be21 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -30,7 +30,7 @@ enable_ls_proxy, enable_ton_storage, enable_ton_storage_provider, - EnableMode + EnableMode, ConfigureFromBackup ) from mytoninstaller.config import ( CreateLocalConfig, @@ -276,6 +276,16 @@ def General(local, console): mx = sys.argv.index("-m") mode = sys.argv[mx+1] local.buffer.mode = mode + if "--only-mtc" in sys.argv: + ox = sys.argv.index("--only-mtc") + local.buffer.only_mtc = str2bool(sys.argv[ox+1]) + if "--only-node" in sys.argv: + pass + if "--backup" in sys.argv: + bx = sys.argv.index("--backup") + backup = sys.argv[bx+1] + if backup != "none": + local.buffer.backup = backup #end if FirstMytoncoreSettings(local) @@ -286,9 +296,16 @@ def General(local, console): BackupMconfig(local) CreateSymlinks(local) EnableMode(local) + ConfigureFromBackup(local) #end define +""" +Node server: Mtc is being installed and configures node, creates backup but does not create mytoncore service. +Mtc server: TON bins and Mtc are being installed, restores keys and mytoncore db from backup (in case validator on Node server was installed before), creates mytoncore service. +""" + + ### ### Start of the program ### diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index cba219ea..880fb34c 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -22,6 +22,9 @@ def FirstNodeSettings(local): + if local.buffer.only_mtc: + return + local.add_log("start FirstNodeSettings fuction", "debug") # Создать переменные @@ -300,6 +303,9 @@ def EnableValidatorConsole(local): #end define def EnableLiteServer(local): + if local.buffer.only_mtc: + return + local.add_log("start EnableLiteServer function", "debug") # Create variables @@ -912,3 +918,17 @@ def EnableMode(local): return args = ["su", "-l", local.buffer.user, "-c", ' '.join(args)] subprocess.run(args) + + +def ConfigureFromBackup(local): + from mytoncore import MyTonCore + from mypylib.mypylib import MyPyClass # todo move to file header + from modules.backups import BackupModule + if not local.buffer.backup: + return + local.add_log("start ConfigureFromBackup function", "info") + backup_file = local.buffer.backup + + ton = MyTonCore(MyPyClass('mytoncore.py')) + module = BackupModule(ton, local) + module.restore_backup([backup_file, '-y']) diff --git a/scripts/install.sh b/scripts/install.sh index 1da6e42c..ce3a33e2 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -45,10 +45,12 @@ config="https://ton-blockchain.github.io/global.config.json" telemetry=true ignore=false dump=false +only_mtc=false +backup=none cpu_required=16 mem_required=64000000 # 64GB in KB -while getopts ":c:tida:r:b:m:n:v:u:h" flag; do +while getopts ":c:tidoa:r:b:m:n:v:u:p:h" flag; do case "${flag}" in c) config=${OPTARG};; t) telemetry=false;; @@ -61,6 +63,8 @@ while getopts ":c:tida:r:b:m:n:v:u:h" flag; do n) network=${OPTARG};; v) ton_node_version=${OPTARG};; u) user=${OPTARG};; + o) only_mtc=true;; + p) backup=${OPTARG};; h) show_help_and_exit;; *) echo "Flag -${flag} is not recognized. Aborting" @@ -144,7 +148,7 @@ if [ "${user}" = "" ]; then # no user fi fi echo "User: $user" -python3 -m mytoninstaller -u ${user} -t ${telemetry} --dump ${dump} -m ${mode} +python3 -m mytoninstaller -u ${user} -t ${telemetry} --dump ${dump} -m ${mode} --only-mtc ${only_mtc} --backup ${backup} # set migrate version migrate_version=1 From 7073820caa123831d6695e71b2383357e5c9b86c Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 12 Dec 2024 14:12:12 +0900 Subject: [PATCH 136/206] patch work dir on installing from backup --- mytoninstaller/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 880fb34c..cd729dbe 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -926,9 +926,12 @@ def ConfigureFromBackup(local): from modules.backups import BackupModule if not local.buffer.backup: return + mconfig_path = local.buffer.mconfig_path + mconfig_dir = get_dir_from_path(mconfig_path) local.add_log("start ConfigureFromBackup function", "info") backup_file = local.buffer.backup ton = MyTonCore(MyPyClass('mytoncore.py')) + ton.local.buffer.my_work_dir = mconfig_dir module = BackupModule(ton, local) module.restore_backup([backup_file, '-y']) From 972e267dc560db0c869a34779762374d7e7d8cb3 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 12 Dec 2024 14:55:36 +0900 Subject: [PATCH 137/206] require backup in only_mtc installation mode --- scripts/install.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/install.sh b/scripts/install.sh index ce3a33e2..efeea41d 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -73,7 +73,13 @@ while getopts ":c:tidoa:r:b:m:n:v:u:p:h" flag; do done -if [ "${mode}" = "" ]; then # no mode +if [ "$only_mtc" = true ] && [ "$backup" = "none" ]; then + echo "Backup file must be provided if only mtc installation" + exit 1 +fi + + +if [ "${mode}" = "" ] && [ "$backup" = "none" ]; then # no mode or backup was provided echo "Running cli installer" wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/install.py pip3 install inquirer==3.4.0 From 2af55336947ca29a74a328ec9a26c69ff34e9f25 Mon Sep 17 00:00:00 2001 From: yungwine Date: Sat, 14 Dec 2024 01:48:11 +0900 Subject: [PATCH 138/206] fix node install --- scripts/ton_installer.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/ton_installer.sh b/scripts/ton_installer.sh index 6454fd2a..5dc962cf 100644 --- a/scripts/ton_installer.sh +++ b/scripts/ton_installer.sh @@ -118,8 +118,10 @@ if [ "${ton_node_version}" != "master" ]; then cd ../ fi +cd $SOURCES_DIR/ton git submodule sync --recursive git submodule update +cd ../ git config --global --add safe.directory $SOURCES_DIR/ton From 7128e9cdd5c57a90269f05b5ad7c096102efeaea Mon Sep 17 00:00:00 2001 From: yungwine Date: Sat, 14 Dec 2024 01:45:33 +0900 Subject: [PATCH 139/206] add setting external ip if only_mtc --- mytonctrl/scripts/restore_backup.sh | 9 ++++++-- mytoninstaller/settings.py | 35 ++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index ca53f0de..972596bd 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -48,9 +48,14 @@ echo -e "${COLOR}[2/4]${ENDC} Extracted files from archive" rm -r /var/ton-work/db/dht-* -python3 -c "import json;path='/var/ton-work/db/config.json';f=open(path);d=json.load(f);f.close();d['addrs'][0]['ip']=int($ip);f=open(path, 'w');f.write(json.dumps(d, indent=4));f.close()" +if [ $ip -ne 0 ]; then + echo "Replacing IP in node config" + python3 -c "import json;path='/var/ton-work/db/config.json';f=open(path);d=json.load(f);f.close();d['addrs'][0]['ip']=int($ip);f=open(path, 'w');f.write(json.dumps(d, indent=4));f.close()" +else + echo "IP is not provided, skipping IP replacement" +fi -echo -e "${COLOR}[3/4]${ENDC} Deleted DHT files, replaced IP in node config" +echo -e "${COLOR}[3/4]${ENDC} Deleted DHT files" systemctl start validator systemctl start mytoncore diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index cd729dbe..26d9044b 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -14,7 +14,7 @@ run_as_root, color_print, ip2int, - Dict + Dict, int2ip ) from mytoninstaller.utils import StartValidator, StartMytoncore, start_service, stop_service, get_ed25519_pubkey from mytoninstaller.config import SetConfig, GetConfig, get_own_ip, backup_config @@ -201,6 +201,8 @@ def FirstMytoncoreSettings(local): #end define def EnableValidatorConsole(local): + if local.buffer.only_mtc: + return local.add_log("start EnableValidatorConsole function", "debug") # Create variables @@ -920,12 +922,25 @@ def EnableMode(local): subprocess.run(args) +def set_external_ip(local, ip): + mconfig_path = local.buffer.mconfig_path + + mconfig = GetConfig(path=mconfig_path) + + mconfig.liteClient.liteServer.ip = ip + mconfig.validatorConsole.addr = f'{ip}:{mconfig.validatorConsole.addr.split(":")[1]}' + + # write mconfig + local.add_log("write mconfig", "debug") + SetConfig(path=mconfig_path, data=mconfig) + + def ConfigureFromBackup(local): + if not local.buffer.backup: + return from mytoncore import MyTonCore from mypylib.mypylib import MyPyClass # todo move to file header from modules.backups import BackupModule - if not local.buffer.backup: - return mconfig_path = local.buffer.mconfig_path mconfig_dir = get_dir_from_path(mconfig_path) local.add_log("start ConfigureFromBackup function", "info") @@ -935,3 +950,17 @@ def ConfigureFromBackup(local): ton.local.buffer.my_work_dir = mconfig_dir module = BackupModule(ton, local) module.restore_backup([backup_file, '-y']) + + if local.buffer.only_mtc: + local.add_log("Installing only mtc", "info") + vconfig_path = local.buffer.vconfig_path + vconfig = GetConfig(path=vconfig_path) + try: + node_ip = int2ip(vconfig['addrs'][0]['ip']) + except: + local.add_log("Can't get ip from validator", "error") + return + user = local.buffer.user + args = ["chown", '-R', user + ':' + user, local.buffer.keys_dir] + subprocess.run(args) + set_external_ip(local, node_ip) From 9114bb2e7d9b7721fb3cb27efa2356f75196f0a3 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 16 Dec 2024 13:21:59 +0900 Subject: [PATCH 140/206] update restoring backup --- modules/backups.py | 8 ++++++-- mytoninstaller/settings.py | 8 ++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/backups.py b/modules/backups.py index 7ed70954..149083bf 100644 --- a/modules/backups.py +++ b/modules/backups.py @@ -46,6 +46,11 @@ def create_backup(self, args): return process.returncode # end define + @staticmethod + def run_restore_backup(args): + restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh') + return run_as_root(["bash", restore_script_path] + args) + def restore_backup(self, args): if len(args) == 0 or len(args) > 2: color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") @@ -67,8 +72,7 @@ def restore_backup(self, args): ip = str(ip2int(get_own_ip())) command_args = ["-m", self.ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip] - restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh') - if run_as_root(["bash", restore_script_path] + command_args) == 0: + if self.run_restore_backup(command_args) == 0: color_print("restore_backup - {green}OK{endc}") self.local.exit() else: diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 26d9044b..769e0b4c 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -938,18 +938,14 @@ def set_external_ip(local, ip): def ConfigureFromBackup(local): if not local.buffer.backup: return - from mytoncore import MyTonCore - from mypylib.mypylib import MyPyClass # todo move to file header from modules.backups import BackupModule mconfig_path = local.buffer.mconfig_path mconfig_dir = get_dir_from_path(mconfig_path) local.add_log("start ConfigureFromBackup function", "info") backup_file = local.buffer.backup - ton = MyTonCore(MyPyClass('mytoncore.py')) - ton.local.buffer.my_work_dir = mconfig_dir - module = BackupModule(ton, local) - module.restore_backup([backup_file, '-y']) + os.makedirs(local.buffer.ton_work_dir, exist_ok=True) + BackupModule.run_restore_backup(["-m", mconfig_dir, "-n", backup_file]) if local.buffer.only_mtc: local.add_log("Installing only mtc", "info") From e61f3b59656b48e9a20aa2f2d544c5d9b3ef4163 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 16 Dec 2024 20:30:43 +0900 Subject: [PATCH 141/206] add --only-node flag --- modules/backups.py | 8 ++++++-- mytoninstaller/mytoninstaller.py | 6 ++++-- mytoninstaller/settings.py | 17 +++++++++++++++++ scripts/install.sh | 9 +++++++-- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/modules/backups.py b/modules/backups.py index 149083bf..0458c298 100644 --- a/modules/backups.py +++ b/modules/backups.py @@ -27,6 +27,11 @@ def create_tmp_ton_dir(self): self.create_keyring(dir_name_db) return dir_name + @staticmethod + def run_create_backup(args): + backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh') + return subprocess.run(["bash", backup_script_path] + args, timeout=5) + def create_backup(self, args): if len(args) > 1: color_print("{red}Bad args. Usage:{endc} create_backup [filename]") @@ -35,8 +40,7 @@ def create_backup(self, args): command_args = ["-m", self.ton.local.buffer.my_work_dir, "-t", tmp_dir] if len(args) == 1: command_args += ["-d", args[0]] - backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh') - process = subprocess.run(["bash", backup_script_path] + command_args, timeout=5) + process = self.run_create_backup(command_args) if process.returncode == 0: color_print("create_backup - {green}OK{endc}") diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index 2d54be21..2df94c3a 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -30,7 +30,7 @@ enable_ls_proxy, enable_ton_storage, enable_ton_storage_provider, - EnableMode, ConfigureFromBackup + EnableMode, ConfigureFromBackup, ConfigureOnlyNode ) from mytoninstaller.config import ( CreateLocalConfig, @@ -280,7 +280,8 @@ def General(local, console): ox = sys.argv.index("--only-mtc") local.buffer.only_mtc = str2bool(sys.argv[ox+1]) if "--only-node" in sys.argv: - pass + ox = sys.argv.index("--only-node") + local.buffer.only_node = str2bool(sys.argv[ox+1]) if "--backup" in sys.argv: bx = sys.argv.index("--backup") backup = sys.argv[bx+1] @@ -297,6 +298,7 @@ def General(local, console): CreateSymlinks(local) EnableMode(local) ConfigureFromBackup(local) + ConfigureOnlyNode(local) #end define diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 769e0b4c..f4fb6fe1 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -960,3 +960,20 @@ def ConfigureFromBackup(local): args = ["chown", '-R', user + ':' + user, local.buffer.keys_dir] subprocess.run(args) set_external_ip(local, node_ip) + + +def ConfigureOnlyNode(local): + if not local.buffer.only_node: + return + from modules.backups import BackupModule + mconfig_path = local.buffer.mconfig_path + mconfig_dir = get_dir_from_path(mconfig_path) + local.add_log("start ConfigureOnlyNode function", "info") + + process = BackupModule.run_create_backup(["-m", mconfig_dir, ]) + if process.returncode != 0: + local.add_log("Backup creation failed", "error") + return + local.add_log("Backup successfully created. Use this file on the controller server with `--only-mtc` flag on installation.", "info") + + stop_service(local, 'mytoncore') diff --git a/scripts/install.sh b/scripts/install.sh index efeea41d..fb5d8863 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -32,6 +32,9 @@ show_help_and_exit() { echo ' -n NETWORK Specify the network (mainnet or testnet)' echo ' -v VERSION Specify the ton node version (commit, branch, or tag)' echo ' -u USER Specify the user to be used for MyTonCtrl installation' + echo ' -p PATH Provide backup file for MyTonCtrl installation' + echo ' -o Install only MyTonCtrl. Must be used with -p' + echo ' -l Install only TON node' echo ' -h Show this help' exit } @@ -46,11 +49,12 @@ telemetry=true ignore=false dump=false only_mtc=false +only_node=false backup=none cpu_required=16 mem_required=64000000 # 64GB in KB -while getopts ":c:tidoa:r:b:m:n:v:u:p:h" flag; do +while getopts ":c:tidola:r:b:m:n:v:u:p:h" flag; do case "${flag}" in c) config=${OPTARG};; t) telemetry=false;; @@ -64,6 +68,7 @@ while getopts ":c:tidoa:r:b:m:n:v:u:p:h" flag; do v) ton_node_version=${OPTARG};; u) user=${OPTARG};; o) only_mtc=true;; + l) only_node=true;; p) backup=${OPTARG};; h) show_help_and_exit;; *) @@ -154,7 +159,7 @@ if [ "${user}" = "" ]; then # no user fi fi echo "User: $user" -python3 -m mytoninstaller -u ${user} -t ${telemetry} --dump ${dump} -m ${mode} --only-mtc ${only_mtc} --backup ${backup} +python3 -m mytoninstaller -u ${user} -t ${telemetry} --dump ${dump} -m ${mode} --only-mtc ${only_mtc} --backup ${backup} --only-node ${only_node} # set migrate version migrate_version=1 From b764a2e259e39290d41a5aa208489ec36e4c547f Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 18 Dec 2024 19:41:52 +0900 Subject: [PATCH 142/206] add onlyNode flag --- modules/__init__.py | 1 + mytoncore/functions.py | 10 +++++++--- mytoninstaller/settings.py | 6 +++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/modules/__init__.py b/modules/__init__.py index 3d09bb9e..076e12d0 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -61,6 +61,7 @@ class Setting: 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id'), 'auto_backup': Setting('validator', None, 'Make validator backup every election'), 'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'), + 'onlyNode': Setting(None, None, 'MyTonCtrl will work only for collecting validator telemetry (if `sendTelemetry` is True), without participating in Elections and etc.') } diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 0dd136ff..00936ca9 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -551,8 +551,14 @@ def General(local): # scanner.Run() # Start threads - local.start_cycle(Elections, sec=600, args=(local, ton, )) local.start_cycle(Statistics, sec=10, args=(local, )) + local.start_cycle(Telemetry, sec=60, args=(local, ton, )) + local.start_cycle(OverlayTelemetry, sec=7200, args=(local, ton, )) + if local.db.get("onlyNode"): # mytoncore service works only for telemetry + thr_sleep() + return + + local.start_cycle(Elections, sec=600, args=(local, ton, )) local.start_cycle(Offers, sec=600, args=(local, ton, )) local.start_cycle(save_past_events, sec=300, args=(local, ton, )) @@ -562,8 +568,6 @@ def General(local): local.start_cycle(Complaints, sec=t, args=(local, ton, )) local.start_cycle(Slashing, sec=t, args=(local, ton, )) - local.start_cycle(Telemetry, sec=60, args=(local, ton, )) - local.start_cycle(OverlayTelemetry, sec=7200, args=(local, ton, )) local.start_cycle(ScanLiteServers, sec=60, args=(local, ton,)) from modules.custom_overlays import CustomOverlayModule diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index f4fb6fe1..228f9b87 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -976,4 +976,8 @@ def ConfigureOnlyNode(local): return local.add_log("Backup successfully created. Use this file on the controller server with `--only-mtc` flag on installation.", "info") - stop_service(local, 'mytoncore') + mconfig = GetConfig(path=mconfig_path) + mconfig.onlyNode = True + SetConfig(path=mconfig_path, data=mconfig) + + start_service(local, 'mytoncore') From 7a88e283ca367c85ccc13254bb10ea0740c8d8ca Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 20 Dec 2024 09:25:34 +0400 Subject: [PATCH 143/206] add node ip address to status if node is remote --- modules/alert_bot.py | 2 +- mytoncore/mytoncore.py | 5 ++++- mytonctrl/mytonctrl.py | 8 +++++++- mytonctrl/resources/translate.json | 5 +++++ mytoninstaller/mytoninstaller.py | 6 ------ 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 2ebf4034..b1c1d13c 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -142,7 +142,7 @@ def init(self): from modules.validator import ValidatorModule self.validator_module = ValidatorModule(self.ton, self.local) self.hostname = get_hostname() - self.ip = self.ton.get_validator_engine_ip() + self.ip = self.ton.get_node_ip() self.set_global_vars() self.inited = True diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 3aac3c0b..f526a939 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3819,13 +3819,16 @@ def GetNetworkName(self): return "unknown" #end define - def get_validator_engine_ip(self): + def get_node_ip(self): try: config = self.GetValidatorConfig() return int2ip(config['addrs'][0]['ip']) except: return None + def get_validator_engine_ip(self): + return self.validatorConsole.addr.split(':')[0] + def GetFunctionBuffer(self, name, timeout=10): timestamp = get_timestamp() buff = self.local.buffer.get(name) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 3263b3b2..7b939e0c 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -762,6 +762,11 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, validatorVersion_text = local.translate("local_status_version_validator").format(validatorGitHash_text, validatorGitBranch_text) color_print(local.translate("local_status_head")) + node_ip = ton.get_validator_engine_ip() + is_node_remote = node_ip != '127.0.0.1' + if is_node_remote: + nodeIpAddr_text = local.translate("node_ip_address").format(node_ip) + color_print(nodeIpAddr_text) if ton.using_validator(): print(validatorIndex_text) # print(validatorEfficiency_text) @@ -776,7 +781,8 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, print(disksLoad_text) print(mytoncoreStatus_text) - print(validatorStatus_text) + if not is_node_remote: + print(validatorStatus_text) print(validator_out_of_sync_text) print(validator_out_of_ser_text) print(dbStatus_text) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 83752783..ee3f900f 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -249,6 +249,11 @@ "ru": "{cyan}===[ Статус ноды ]==={endc}", "zh_TW": "{cyan}===[ 节点狀態 ]==={endc}" }, + "node_ip_address": { + "en": "Node IP address: {{bold}}{0}{{endc}}", + "ru": "IP адрес Ноды: {{bold}}{0}{{endc}}", + "zh_TW": "節點 IP 地址: {{bold}}{0}{{endc}}" + }, "local_status_validator_index": { "en": "Validator index: {0}", "ru": "Индекс валидатора: {0}", diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index 2df94c3a..cb0dffec 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -302,12 +302,6 @@ def General(local, console): #end define -""" -Node server: Mtc is being installed and configures node, creates backup but does not create mytoncore service. -Mtc server: TON bins and Mtc are being installed, restores keys and mytoncore db from backup (in case validator on Node server was installed before), creates mytoncore service. -""" - - ### ### Start of the program ### From 5aae0080e83878d91c5b45a55d0015e4a54c82c8 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 23 Dec 2024 16:09:32 +0400 Subject: [PATCH 144/206] add default value to mode on install --- mytoninstaller/settings.py | 2 +- scripts/install.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 228f9b87..14566fc5 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -914,7 +914,7 @@ def CreateSymlinks(local): def EnableMode(local): args = ["python3", "-m", "mytoncore", "-e"] - if local.buffer.mode: + if local.buffer.mode and local.buffer.mode != "none": args.append("enable_mode_" + local.buffer.mode) else: return diff --git a/scripts/install.sh b/scripts/install.sh index fb5d8863..68abf777 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -51,6 +51,7 @@ dump=false only_mtc=false only_node=false backup=none +mode=none cpu_required=16 mem_required=64000000 # 64GB in KB @@ -84,7 +85,7 @@ if [ "$only_mtc" = true ] && [ "$backup" = "none" ]; then fi -if [ "${mode}" = "" ] && [ "$backup" = "none" ]; then # no mode or backup was provided +if [ "${mode}" = "none" ] && [ "$backup" = "none" ]; then # no mode or backup was provided echo "Running cli installer" wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/install.py pip3 install inquirer==3.4.0 From 0f90ecd90cec721c57c32ca1cc20792f96c31861 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 24 Dec 2024 11:11:57 +0400 Subject: [PATCH 145/206] fix installing from backup --- mytoninstaller/settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 14566fc5..3276b720 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -945,9 +945,12 @@ def ConfigureFromBackup(local): backup_file = local.buffer.backup os.makedirs(local.buffer.ton_work_dir, exist_ok=True) - BackupModule.run_restore_backup(["-m", mconfig_dir, "-n", backup_file]) + if not local.buffer.only_mtc: + ip = str(ip2int(get_own_ip())) + BackupModule.run_restore_backup(["-m", mconfig_dir, "-n", backup_file, "-i", ip]) if local.buffer.only_mtc: + BackupModule.run_restore_backup(["-m", mconfig_dir, "-n", backup_file]) local.add_log("Installing only mtc", "info") vconfig_path = local.buffer.vconfig_path vconfig = GetConfig(path=vconfig_path) From 6595747749295e81402293e0eb13a4ad5cade3aa Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 24 Dec 2024 14:21:57 +0400 Subject: [PATCH 146/206] fix cp in restore_backup --- mytonctrl/scripts/restore_backup.sh | 14 ++++++++++---- mytoninstaller/settings.py | 3 --- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index 972596bd..005ee7d8 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -1,13 +1,15 @@ name="backup.tar.gz" mtc_dir="$HOME/.local/share/mytoncore" ip=0 +user=$(logname) # Get arguments -while getopts n:m:i: flag +while getopts n:m:i:u: flag do case "${flag}" in n) name=${OPTARG};; m) mtc_dir=${OPTARG};; i) ip=${OPTARG};; + u) user=${OPTARG};; *) echo "Flag -${flag} is not recognized. Aborting" exit 1 ;; @@ -38,9 +40,13 @@ if [ ! -d ${tmp_dir}/db ]; then fi rm -rf /var/ton-work/db/keyring -cp -rf ${tmp_dir}/db /var/ton-work -cp -rf ${tmp_dir}/keys /var/ton-work -cp -rfT ${tmp_dir}/mytoncore $mtc_dir + +chown -R $user:$user ${tmp_dir}/mytoncore +chown -R $user:$user ${tmp_dir}/keys + +cp -rfp ${tmp_dir}/db /var/ton-work +cp -rfp ${tmp_dir}/keys /var/ton-work +cp -rfpT ${tmp_dir}/mytoncore $mtc_dir chown -R validator:validator /var/ton-work/db/keyring diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 3276b720..43dafd3f 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -959,9 +959,6 @@ def ConfigureFromBackup(local): except: local.add_log("Can't get ip from validator", "error") return - user = local.buffer.user - args = ["chown", '-R', user + ':' + user, local.buffer.keys_dir] - subprocess.run(args) set_external_ip(local, node_ip) From 44f34b379169c4d6ac8955f37156178d890b74bf Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 25 Dec 2024 16:57:46 +0400 Subject: [PATCH 147/206] update update.sh --- mytonctrl/scripts/update.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mytonctrl/scripts/update.sh b/mytonctrl/scripts/update.sh index 8bd4ecd9..c9de406b 100644 --- a/mytonctrl/scripts/update.sh +++ b/mytonctrl/scripts/update.sh @@ -12,7 +12,7 @@ author="ton-blockchain" repo="mytonctrl" branch="master" srcdir="/usr/src/" -bindir="/usr/bin/" +tmpdir="/tmp/mytonctrl_src/" # Get arguments while getopts a:r:b: flag @@ -28,16 +28,18 @@ done COLOR='\033[92m' ENDC='\033[0m' -# Go to work dir -cd ${srcdir} +mkdir -p ${tmpdir} +cd ${tmpdir} +rm -rf ${tmpdir}/${repo} +echo "https://github.com/${author}/${repo}.git -> ${branch}" +git clone --recursive https://github.com/${author}/${repo}.git || exit 1 -# uninstall previous version rm -rf ${srcdir}/${repo} pip3 uninstall -y mytonctrl # Update code -echo "https://github.com/${author}/${repo}.git -> ${branch}" -git clone --recursive https://github.com/${author}/${repo}.git +cd ${srcdir} +cp -rf ${tmpdir}/${repo} ${srcdir} cd ${repo} && git checkout ${branch} pip3 install -U . From a7c6c027b94d8cef8fc88e3ac7f68207c17dc037 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 26 Dec 2024 12:07:25 +0400 Subject: [PATCH 148/206] update upgrade.sh --- mytonctrl/scripts/upgrade.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mytonctrl/scripts/upgrade.sh b/mytonctrl/scripts/upgrade.sh index c3c4c52b..fe952562 100644 --- a/mytonctrl/scripts/upgrade.sh +++ b/mytonctrl/scripts/upgrade.sh @@ -13,6 +13,7 @@ repo="ton" branch="master" srcdir="/usr/src/" bindir="/usr/bin/" +tmpdir="/tmp/ton_src/" # Get arguments while getopts a:r:b: flag @@ -59,13 +60,18 @@ else opensslPath=${bindir}/openssl_3 fi +rm -rf ${tmpdir}/${repo} +mkdir -p ${tmpdir}/${repo} +cd ${tmpdir}/${repo} +echo "https://github.com/${author}/${repo}.git -> ${branch}" +git clone --recursive https://github.com/${author}/${repo}.git . || exit 1 + # Go to work dir cd ${srcdir}/${repo} ls -A1 | xargs rm -rf # Update code -echo "https://github.com/${author}/${repo}.git -> ${branch}" -git clone --recursive https://github.com/${author}/${repo}.git . +cp -rfT ${tmpdir}/${repo} . git checkout ${branch} git submodule sync --recursive From 64ecaf6aa7b2012208b906ae8639dbe63fa20909 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 30 Dec 2024 11:01:06 +0400 Subject: [PATCH 149/206] add pip conf on new installations --- scripts/install.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/install.sh b/scripts/install.sh index 1da6e42c..6ba21f15 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -72,7 +72,7 @@ done if [ "${mode}" = "" ]; then # no mode echo "Running cli installer" wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/install.py - pip3 install inquirer==3.4.0 + pip3 install inquirer==3.4.0 --break-system-packages python3 install.py exit fi @@ -107,6 +107,16 @@ if [[ "$OSTYPE" =~ darwin.* ]]; then mkdir -p ${SOURCES_DIR} fi + +if [ ! -f ~/.config/pip/pip.conf ]; then # create pip config + mkdir -p ~/.config/pip +cat > ~/.config/pip/pip.conf < Date: Wed, 8 Jan 2025 16:36:19 +0400 Subject: [PATCH 150/206] add stake info alerting --- modules/alert_bot.py | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 2ebf4034..5b99a7a5 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -18,6 +18,7 @@ class Alert: HOUR = 3600 VALIDATION_PERIOD = 65536 FREEZE_PERIOD = 32768 +ELECTIONS_START_BEFORE = 8192 ALERTS = { @@ -70,6 +71,16 @@ class Alert: "Validator has been slashed in previous round for {amount} TON", FREEZE_PERIOD ), + "stake_not_accepted": Alert( + "high", + "Validator's stake has not been accepted", + ELECTIONS_START_BEFORE + ), + "stake_accepted": Alert( + "info", + "Validator's stake {stake} TON has been accepted", + ELECTIONS_START_BEFORE + ) } @@ -128,9 +139,10 @@ def send_alert(self, alert_name: str, *args, **kwargs): def set_global_vars(self): # set global vars for correct alerts timeouts for current network config15 = self.ton.GetConfig15() - global VALIDATION_PERIOD, FREEZE_PERIOD + global VALIDATION_PERIOD, FREEZE_PERIOD, ELECTIONS_START_BEFORE VALIDATION_PERIOD = config15["validatorsElectedFor"] FREEZE_PERIOD = config15["stakeHeldFor"] + ELECTIONS_START_BEFORE = config15["electionsStartBefore"] def init(self): if not self.ton.get_mode_value('alert-bot'): @@ -271,6 +283,32 @@ def check_adnl_connection_failed(self): if not ok: self.send_alert("adnl_connection_failed") + def get_myself_from_election(self): + config = self.ton.GetConfig36() + if not config["validators"]: + return + validator = self.validator_module.find_myself(config["validators"]) + if validator is None: + return False + save_elections = self.ton.GetSaveElections() + elections = save_elections.get(str(config["startWorkTime"])) + if elections is None: + return validator + adnl = self.ton.GetAdnlAddr() + validator['stake'] = elections[adnl].get('stake') + return validator + + def check_stake(self): + if not self.ton.using_validator(): + return + res = self.get_myself_from_election() + if res is None: + return + if res is False: + self.send_alert("stake_not_accepted") + return + self.send_alert("stake_accepted", stake=res.get('stake')) + def check_status(self): if not self.ton.using_alert_bot(): return @@ -285,6 +323,7 @@ def check_status(self): self.local.try_function(self.check_sync) self.local.try_function(self.check_slashed) self.local.try_function(self.check_adnl_connection_failed) + self.local.try_function(self.check_stake) def add_console_commands(self, console): console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) From 68c459f4a17e2af3dd69f8d8f3e12dd9da0baa5f Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 9 Jan 2025 12:06:44 +0400 Subject: [PATCH 151/206] send info alerts without sound --- modules/alert_bot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 5b99a7a5..676f0fac 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -99,13 +99,13 @@ def __init__(self, ton, local, *args, **kwargs): self.chat_id = None self.last_db_check = 0 - def send_message(self, text: str): + def send_message(self, text: str, silent: bool = False): if self.token is None: raise Exception("send_message error: token is not initialized") if self.chat_id is None: raise Exception("send_message error: chat_id is not initialized") request_url = f"https://api.telegram.org/bot{self.token}/sendMessage" - data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML'} + data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML', 'disable_notification': silent} response = requests.post(request_url, data=data, timeout=3) if response.status_code != 200: raise Exception(f"send_message error: {response.text}") @@ -133,7 +133,7 @@ def send_alert(self, alert_name: str, *args, **kwargs):
{alert.text.format(*args, **kwargs)}
''' if time.time() - last_sent > alert.timeout: - self.send_message(text) + self.send_message(text, alert.severity == "info") # send info alerts without sound self.set_alert_sent(alert_name) def set_global_vars(self): From 128ab717b0bbc4d75fea59a3c4d3e3a80ac6d985 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 13 Jan 2025 11:52:17 +0400 Subject: [PATCH 152/206] add prometheus metrics pusher --- modules/__init__.py | 5 +++- modules/prometheus.py | 59 ++++++++++++++++++++++++++++++++++++++++++ mytoncore/functions.py | 3 +++ mytoncore/mytoncore.py | 3 +++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 modules/prometheus.py diff --git a/modules/__init__.py b/modules/__init__.py index 3d09bb9e..5b0b5acf 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -9,6 +9,7 @@ from modules.controller import ControllerModule from modules.liteserver import LiteserverModule from modules.alert_bot import AlertBotModule +from modules.prometheus import PrometheusModule MODES = { @@ -17,7 +18,8 @@ 'single-nominator': SingleNominatorModule, 'liquid-staking': ControllerModule, 'liteserver': LiteserverModule, - 'alert-bot': AlertBotModule + 'alert-bot': AlertBotModule, + 'prometheus': PrometheusModule } @@ -61,6 +63,7 @@ class Setting: 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id'), 'auto_backup': Setting('validator', None, 'Make validator backup every election'), 'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'), + 'prometheus_url': Setting('prometheus', None, 'Prometheus pushgateway url'), } diff --git a/modules/prometheus.py b/modules/prometheus.py new file mode 100644 index 00000000..71fc2492 --- /dev/null +++ b/modules/prometheus.py @@ -0,0 +1,59 @@ +from modules.module import MtcModule +import dataclasses +import requests + + +@dataclasses.dataclass +class Metric: + name: str + description: str + type: str + + def to_format(self, value): + return f""" +# HELP {self.name} {self.description} +# TYPE {self.name} {self.type} +{self.name} {value} +""" + + +METRICS = { + 'master_out_of_sync': Metric('validator_masterchain_out_of_sync_seconds', 'Time difference between current time and timestamp of the last known block', 'gauge'), + 'shard_out_of_sync': Metric('validator_shardchain_out_of_sync_blocks', 'Number of blocks validator\'s shardclient is behind the last known block', 'gauge'), + 'out_of_ser': Metric('validator_out_of_serialization', 'Number of blocks last state serialization was ago', 'gauge'), + 'vc_up': Metric('validator_console_up', 'Is validator\'s validator client up', 'gauge'), +} + + +class PrometheusModule(MtcModule): + + description = 'Prometheus format data exporter' + default_value = False + + def __init__(self, ton, local, *args, **kwargs): + super().__init__(ton, local, *args, **kwargs) + + def get_validator_status_metrics(self): + status = self.ton.GetValidatorStatus() + result = [] + if status.masterchain_out_of_sync is not None: + result.append(METRICS['master_out_of_sync'].to_format(status.masterchain_out_of_sync)) + if status.shardchain_out_of_sync is not None: + result.append(METRICS['shard_out_of_sync'].to_format(status.shardchain_out_of_sync)) + if status.masterchain_out_of_ser is not None: + result.append(METRICS['out_of_ser'].to_format(status.masterchain_out_of_ser)) + result.append(METRICS['vc_up'].to_format(int(status.is_working))) + return result + + def push_metrics(self): + if not self.ton.using_prometheus(): + return + + url = self.ton.local.db.get('prometheus_url') + if url is None: + raise Exception('Prometheus url is not set') + metrics = self.get_validator_status_metrics() + requests.post(url, data='\n'.join(metrics).encode()) + + def add_console_commands(self, console): + ... diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 0dd136ff..96f2abf7 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -572,6 +572,9 @@ def General(local): from modules.alert_bot import AlertBotModule local.start_cycle(AlertBotModule(ton, local).check_status, sec=60, args=()) + from modules.prometheus import PrometheusModule + local.start_cycle(PrometheusModule(ton, local).push_metrics, sec=30, args=()) + thr_sleep() # end define diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 6896bf79..5e2e866a 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3123,6 +3123,9 @@ def using_liteserver(self): def using_alert_bot(self): return self.get_mode_value('alert-bot') + def using_prometheus(self): + return self.get_mode_value('prometheus') + def Tlb2Json(self, text): # Заменить скобки start = 0 From eee0e9deafabefdbc4dc83a1efb92afe812dddd3 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 16 Jan 2025 11:16:42 +0400 Subject: [PATCH 153/206] add testnet dump downloading --- mytoninstaller/settings.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index cba219ea..522047a9 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -88,15 +88,25 @@ def FirstNodeSettings(local): StartValidator(local) #end define +def is_testnet(local): + testnet_zero_state_root_hash = "gj+B8wb/AmlPk1z1AhVI484rhrUpgSr2oSFIh56VoSg=" + with open(local.buffer.global_config_path) as f: + config = json.load(f) + if config['validator']['zero_state']['root_hash'] == testnet_zero_state_root_hash: + return True + return False + def DownloadDump(local): dump = local.buffer.dump - if dump == False: + if dump is False: return #end if local.add_log("start DownloadDump function", "debug") - url = "https://dump.ton.org" - dumpSize = requests.get(url + "/dumps/latest.tar.size.archive.txt").text + url = "https://dump.ton.org/dumps/latest" + if is_testnet(local): + url += '_testnet' + dumpSize = requests.get(url + ".tar.size.archive.txt").text print("dumpSize:", dumpSize) needSpace = int(dumpSize) * 3 diskSpace = psutil.disk_usage("/var") @@ -110,7 +120,7 @@ def DownloadDump(local): # download dump using aria2c to a temporary file temp_file = "/tmp/latest.tar.lz" - cmd = f"aria2c -x 8 -s 8 -c {url}/dumps/latest.tar.lz -d / -o {temp_file}" + cmd = f"aria2c -x 8 -s 8 -c {url}.tar.lz -d / -o {temp_file}" os.system(cmd) # process the downloaded file From 8b080e6f4b029f8de8a94279575551678ca4154e Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 16 Jan 2025 20:27:05 +0400 Subject: [PATCH 154/206] add validator_validation_metrics --- modules/prometheus.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/modules/prometheus.py b/modules/prometheus.py index 71fc2492..31dfb8c7 100644 --- a/modules/prometheus.py +++ b/modules/prometheus.py @@ -19,9 +19,11 @@ def to_format(self, value): METRICS = { 'master_out_of_sync': Metric('validator_masterchain_out_of_sync_seconds', 'Time difference between current time and timestamp of the last known block', 'gauge'), - 'shard_out_of_sync': Metric('validator_shardchain_out_of_sync_blocks', 'Number of blocks validator\'s shardclient is behind the last known block', 'gauge'), + 'shard_out_of_sync': Metric('validator_shardchain_out_of_sync_blocks', 'Number of blocks node\'s shardclient is behind the last known block', 'gauge'), 'out_of_ser': Metric('validator_out_of_serialization', 'Number of blocks last state serialization was ago', 'gauge'), - 'vc_up': Metric('validator_console_up', 'Is validator\'s validator client up', 'gauge'), + 'vc_up': Metric('validator_console_up', 'Is `validator-console` up', 'gauge'), + 'validator_id': Metric('validator_index', 'Validator index', 'gauge'), + 'validator_stake': Metric('validator_stake', 'Validator stake', 'gauge'), } @@ -33,9 +35,8 @@ class PrometheusModule(MtcModule): def __init__(self, ton, local, *args, **kwargs): super().__init__(ton, local, *args, **kwargs) - def get_validator_status_metrics(self): + def get_validator_status_metrics(self, result: list): status = self.ton.GetValidatorStatus() - result = [] if status.masterchain_out_of_sync is not None: result.append(METRICS['master_out_of_sync'].to_format(status.masterchain_out_of_sync)) if status.shardchain_out_of_sync is not None: @@ -43,7 +44,18 @@ def get_validator_status_metrics(self): if status.masterchain_out_of_ser is not None: result.append(METRICS['out_of_ser'].to_format(status.masterchain_out_of_ser)) result.append(METRICS['vc_up'].to_format(int(status.is_working))) - return result + + def get_validator_validation_metrics(self, result: list): + index = self.ton.GetValidatorIndex() + result.append(METRICS['validator_id'].to_format(index)) + config = self.ton.GetConfig34() + save_elections = self.ton.GetSaveElections() + elections = save_elections.get(str(config["startWorkTime"])) + if elections is not None: + adnl = self.ton.GetAdnlAddr() + stake = elections.get(adnl, {}).get('stake') + if stake: + result.append(METRICS['validator_stake'].to_format(round(stake, 2))) def push_metrics(self): if not self.ton.using_prometheus(): @@ -52,7 +64,9 @@ def push_metrics(self): url = self.ton.local.db.get('prometheus_url') if url is None: raise Exception('Prometheus url is not set') - metrics = self.get_validator_status_metrics() + metrics = [] + self.local.try_function(self.get_validator_status_metrics, args=[metrics]) + self.local.try_function(self.get_validator_validation_metrics, args=[metrics]) requests.post(url, data='\n'.join(metrics).encode()) def add_console_commands(self, console): From 207a517d24777cda8120a1d9ef6954ab1fa1aff6 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 17 Jan 2025 13:49:31 +0400 Subject: [PATCH 155/206] add celldb prometheus metrics --- modules/prometheus.py | 10 ++++++++-- mytoncore/mytoncore.py | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/prometheus.py b/modules/prometheus.py index 31dfb8c7..6d1d3114 100644 --- a/modules/prometheus.py +++ b/modules/prometheus.py @@ -23,7 +23,9 @@ def to_format(self, value): 'out_of_ser': Metric('validator_out_of_serialization', 'Number of blocks last state serialization was ago', 'gauge'), 'vc_up': Metric('validator_console_up', 'Is `validator-console` up', 'gauge'), 'validator_id': Metric('validator_index', 'Validator index', 'gauge'), - 'validator_stake': Metric('validator_stake', 'Validator stake', 'gauge'), + 'stake': Metric('validator_stake', 'Validator stake', 'gauge'), + 'celldb_gc_block': Metric('validator_celldb_gc_block', 'Celldb GC block latency', 'gauge'), + 'celldb_gc_state': Metric('validator_celldb_gc_state', 'Celldb GC queue size', 'gauge'), } @@ -43,6 +45,10 @@ def get_validator_status_metrics(self, result: list): result.append(METRICS['shard_out_of_sync'].to_format(status.shardchain_out_of_sync)) if status.masterchain_out_of_ser is not None: result.append(METRICS['out_of_ser'].to_format(status.masterchain_out_of_ser)) + if status.masterchainblock is not None and status.gcmasterchainblock is not None: + result.append(METRICS['celldb_gc_block'].to_format(status.masterchainblock - status.gcmasterchainblock)) + if status.gcmasterchainblock is not None and status.last_deleted_mc_state is not None: + result.append(METRICS['celldb_gc_state'].to_format(status.gcmasterchainblock - status.last_deleted_mc_state)) result.append(METRICS['vc_up'].to_format(int(status.is_working))) def get_validator_validation_metrics(self, result: list): @@ -55,7 +61,7 @@ def get_validator_validation_metrics(self, result: list): adnl = self.ton.GetAdnlAddr() stake = elections.get(adnl, {}).get('stake') if stake: - result.append(METRICS['validator_stake'].to_format(round(stake, 2))) + result.append(METRICS['stake'].to_format(round(stake, 2))) def push_metrics(self): if not self.ton.using_prometheus(): diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 5e2e866a..2be92a34 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -796,6 +796,7 @@ def GetValidatorStatus(self): status.masterchain_out_of_ser = status.masterchainblock - status.stateserializermasterchainseqno status.out_of_sync = status.masterchain_out_of_sync if status.masterchain_out_of_sync > status.shardchain_out_of_sync else status.shardchain_out_of_sync status.out_of_ser = status.masterchain_out_of_ser + status.last_deleted_mc_state = int(parse(result, "last_deleted_mc_state", '\n')) except Exception as ex: self.local.add_log(f"GetValidatorStatus warning: {ex}", "warning") status.is_working = False From 8dcae92b4f6569da8a45dcb1316c0eb27d78ffe8 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 20 Jan 2025 14:13:17 +0400 Subject: [PATCH 156/206] improve checking vc_up metric --- modules/prometheus.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/prometheus.py b/modules/prometheus.py index 6d1d3114..598695ac 100644 --- a/modules/prometheus.py +++ b/modules/prometheus.py @@ -39,6 +39,7 @@ def __init__(self, ton, local, *args, **kwargs): def get_validator_status_metrics(self, result: list): status = self.ton.GetValidatorStatus() + is_working = status.is_working or (status.unixtime is not None) if status.masterchain_out_of_sync is not None: result.append(METRICS['master_out_of_sync'].to_format(status.masterchain_out_of_sync)) if status.shardchain_out_of_sync is not None: @@ -49,7 +50,7 @@ def get_validator_status_metrics(self, result: list): result.append(METRICS['celldb_gc_block'].to_format(status.masterchainblock - status.gcmasterchainblock)) if status.gcmasterchainblock is not None and status.last_deleted_mc_state is not None: result.append(METRICS['celldb_gc_state'].to_format(status.gcmasterchainblock - status.last_deleted_mc_state)) - result.append(METRICS['vc_up'].to_format(int(status.is_working))) + result.append(METRICS['vc_up'].to_format(int(is_working))) def get_validator_validation_metrics(self, result: list): index = self.ton.GetValidatorIndex() From 1e3590ee3270c2d156f4c9674988b116e2165e3a Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 21 Jan 2025 11:16:14 +0400 Subject: [PATCH 157/206] add alerts on stake return --- modules/alert_bot.py | 54 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 676f0fac..0ab807eb 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -80,7 +80,17 @@ class Alert: "info", "Validator's stake {stake} TON has been accepted", ELECTIONS_START_BEFORE - ) + ), + "stake_returned": Alert( + "info", + "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", + 0 + ), + "stake_not_returned": Alert( + "high", + "Validator's stake has not been returned on address {address}.", + 0 + ), } @@ -283,25 +293,28 @@ def check_adnl_connection_failed(self): if not ok: self.send_alert("adnl_connection_failed") - def get_myself_from_election(self): - config = self.ton.GetConfig36() + def get_myself_from_election(self, config: dict): if not config["validators"]: return - validator = self.validator_module.find_myself(config["validators"]) - if validator is None: - return False + adnl = self.ton.GetAdnlAddr() save_elections = self.ton.GetSaveElections() elections = save_elections.get(str(config["startWorkTime"])) if elections is None: - return validator - adnl = self.ton.GetAdnlAddr() + return + if adnl not in elections: # didn't participate in elections + return + validator = self.validator_module.find_myself(config["validators"]) + if validator is None: + return False validator['stake'] = elections[adnl].get('stake') + validator['walletAddr'] = elections[adnl].get('walletAddr') return validator - def check_stake(self): + def check_stake_sent(self): if not self.ton.using_validator(): return - res = self.get_myself_from_election() + config = self.ton.GetConfig36() + res = self.get_myself_from_election(config) if res is None: return if res is False: @@ -309,6 +322,24 @@ def check_stake(self): return self.send_alert("stake_accepted", stake=res.get('stake')) + def check_stake_returned(self): + if not self.ton.using_validator(): + return + config = self.ton.GetConfig32() + if not (config['endWorkTime'] + FREEZE_PERIOD + 1200 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1260): # check between 20th and 21st minutes after stakes have been unfrozen + return + res = self.get_myself_from_election(config) + if not res: + return + trs = self.ton.GetAccountHistory(self.ton.GetAccount(res["walletAddr"]), limit=10) + + for tr in trs: + if tr.time >= config['endWorkTime'] + FREEZE_PERIOD and tr.srcAddr == '3333333333333333333333333333333333333333333333333333333333333333' and tr.body.startswith('F96F7324'): # Elector Recover Stake Response + self.send_alert("stake_returned", stake=res.get('stake'), address=res["walletAddr"], reward=tr.value - res.get('stake', 0)) + return + self.send_alert("stake_not_returned", address=res["walletAddr"]) + + def check_status(self): if not self.ton.using_alert_bot(): return @@ -323,7 +354,8 @@ def check_status(self): self.local.try_function(self.check_sync) self.local.try_function(self.check_slashed) self.local.try_function(self.check_adnl_connection_failed) - self.local.try_function(self.check_stake) + self.local.try_function(self.check_stake_sent) + self.local.try_function(self.check_stake_returned) def add_console_commands(self, console): console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) From b7aca1ccb064d3ad1324d378381874fcef33ca04 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 22 Jan 2025 16:11:03 +0400 Subject: [PATCH 158/206] round values in stake alerts --- modules/alert_bot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 0ab807eb..c3a4cfae 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -320,7 +320,7 @@ def check_stake_sent(self): if res is False: self.send_alert("stake_not_accepted") return - self.send_alert("stake_accepted", stake=res.get('stake')) + self.send_alert("stake_accepted", stake=round(res.get('stake'), 2)) def check_stake_returned(self): if not self.ton.using_validator(): @@ -335,7 +335,7 @@ def check_stake_returned(self): for tr in trs: if tr.time >= config['endWorkTime'] + FREEZE_PERIOD and tr.srcAddr == '3333333333333333333333333333333333333333333333333333333333333333' and tr.body.startswith('F96F7324'): # Elector Recover Stake Response - self.send_alert("stake_returned", stake=res.get('stake'), address=res["walletAddr"], reward=tr.value - res.get('stake', 0)) + self.send_alert("stake_returned", stake=round(tr.value, 2), address=res["walletAddr"], reward=round(tr.value - res.get('stake', 0), 2)) return self.send_alert("stake_not_returned", address=res["walletAddr"]) From 2c1a238a6bf47de7f3f35b3fcf9c183d80e8c35b Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 23 Jan 2025 13:13:40 +0400 Subject: [PATCH 159/206] add voting alert --- modules/alert_bot.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index c3a4cfae..c7348a23 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -84,12 +84,17 @@ class Alert: "stake_returned": Alert( "info", "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", - 0 + 60 ), "stake_not_returned": Alert( "high", "Validator's stake has not been returned on address {address}.", - 0 + 60 + ), + "voting": Alert( + "high", + "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.", + VALIDATION_PERIOD ), } @@ -339,6 +344,19 @@ def check_stake_returned(self): return self.send_alert("stake_not_returned", address=res["walletAddr"]) + def check_voting(self): + if not self.ton.using_validator(): + return + validator_index = self.ton.GetValidatorIndex() + if validator_index == -1: + return + need_to_vote = [] + offers = self.ton.GetOffers() + for offer in offers: + if not offer['isPassed'] and offer['approvedPercent'] >= 50 and validator_index not in offer['votedValidators']: + need_to_vote.append(offer['hash']) + if need_to_vote: + self.send_alert("voting", hashes=' '.join(need_to_vote)) def check_status(self): if not self.ton.using_alert_bot(): @@ -356,6 +374,7 @@ def check_status(self): self.local.try_function(self.check_adnl_connection_failed) self.local.try_function(self.check_stake_sent) self.local.try_function(self.check_stake_returned) + self.local.try_function(self.check_voting) def add_console_commands(self, console): console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) From 20637b63c7a565c16bc00fe8a0f82061b974a488 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 24 Jan 2025 11:46:53 +0400 Subject: [PATCH 160/206] fix alerts timeouts for testnet --- modules/alert_bot.py | 160 ++++++++++++++++++++++--------------------- 1 file changed, 83 insertions(+), 77 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index c7348a23..c1ee2867 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -21,82 +21,87 @@ class Alert: ELECTIONS_START_BEFORE = 8192 -ALERTS = { - "low_wallet_balance": Alert( - "low", - "Validator wallet {wallet} balance is low: {balance} TON.", - 18*HOUR - ), - "db_usage_80": Alert( - "high", - """TON DB usage > 80%. Clean the TON database: - https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming - or (and) set node\'s archive ttl to lower value.""", - 24*HOUR - ), - "db_usage_95": Alert( - "critical", - """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: - https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming - or (and) set node\'s archive ttl to lower value.""", - 6*HOUR - ), - "low_efficiency": Alert( - "high", - """Validator efficiency is low: {efficiency}%.""", - VALIDATION_PERIOD // 3 - ), - "out_of_sync": Alert( - "critical", - "Node is out of sync on {sync} sec.", - 300 - ), - "service_down": Alert( - "critical", - "validator.service is down.", - 300 - ), - "adnl_connection_failed": Alert( - "high", - "ADNL connection to node failed", - 3*HOUR - ), - "zero_block_created": Alert( - "critical", - "Validator has not created any blocks in the last {hours} hours.", - VALIDATION_PERIOD // 3 - ), - "validator_slashed": Alert( - "high", - "Validator has been slashed in previous round for {amount} TON", - FREEZE_PERIOD - ), - "stake_not_accepted": Alert( - "high", - "Validator's stake has not been accepted", - ELECTIONS_START_BEFORE - ), - "stake_accepted": Alert( - "info", - "Validator's stake {stake} TON has been accepted", - ELECTIONS_START_BEFORE - ), - "stake_returned": Alert( - "info", - "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", - 60 - ), - "stake_not_returned": Alert( - "high", - "Validator's stake has not been returned on address {address}.", - 60 - ), - "voting": Alert( - "high", - "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.", - VALIDATION_PERIOD - ), -} +ALERTS = {} + + +def init_alerts(): + global ALERTS + ALERTS = { + "low_wallet_balance": Alert( + "low", + "Validator wallet {wallet} balance is low: {balance} TON.", + 18 * HOUR + ), + "db_usage_80": Alert( + "high", + """TON DB usage > 80%. Clean the TON database: + https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming + or (and) set node\'s archive ttl to lower value.""", + 24 * HOUR + ), + "db_usage_95": Alert( + "critical", + """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: + https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming + or (and) set node\'s archive ttl to lower value.""", + 6 * HOUR + ), + "low_efficiency": Alert( + "high", + """Validator efficiency is low: {efficiency}%.""", + VALIDATION_PERIOD // 3 + ), + "out_of_sync": Alert( + "critical", + "Node is out of sync on {sync} sec.", + 300 + ), + "service_down": Alert( + "critical", + "validator.service is down.", + 300 + ), + "adnl_connection_failed": Alert( + "high", + "ADNL connection to node failed", + 3 * HOUR + ), + "zero_block_created": Alert( + "critical", + "Validator has not created any blocks in the last {hours} hours.", + VALIDATION_PERIOD // 3 + ), + "validator_slashed": Alert( + "high", + "Validator has been slashed in previous round for {amount} TON", + FREEZE_PERIOD + ), + "stake_not_accepted": Alert( + "high", + "Validator's stake has not been accepted", + ELECTIONS_START_BEFORE + ), + "stake_accepted": Alert( + "info", + "Validator's stake {stake} TON has been accepted", + ELECTIONS_START_BEFORE + ), + "stake_returned": Alert( + "info", + "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", + 60 + ), + "stake_not_returned": Alert( + "high", + "Validator's stake has not been returned on address {address}.", + 60 + ), + "voting": Alert( + "high", + "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.", + VALIDATION_PERIOD + ), + } class AlertBotModule(MtcModule): @@ -171,6 +176,7 @@ def init(self): self.hostname = get_hostname() self.ip = self.ton.get_validator_engine_ip() self.set_global_vars() + init_alerts() self.inited = True def get_alert_from_db(self, alert_name: str): @@ -354,7 +360,7 @@ def check_voting(self): offers = self.ton.GetOffers() for offer in offers: if not offer['isPassed'] and offer['approvedPercent'] >= 50 and validator_index not in offer['votedValidators']: - need_to_vote.append(offer['hash']) + need_to_vote.append(offer['hash']) if need_to_vote: self.send_alert("voting", hashes=' '.join(need_to_vote)) From af306067ce8ac6087d08d759ddbdc0f64e1ad55f Mon Sep 17 00:00:00 2001 From: yungwine Date: Sat, 25 Jan 2025 13:02:27 +0400 Subject: [PATCH 161/206] update stake alert --- modules/alert_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index c1ee2867..1e4f5132 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -337,7 +337,7 @@ def check_stake_returned(self): if not self.ton.using_validator(): return config = self.ton.GetConfig32() - if not (config['endWorkTime'] + FREEZE_PERIOD + 1200 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1260): # check between 20th and 21st minutes after stakes have been unfrozen + if not (config['endWorkTime'] + FREEZE_PERIOD + 1800 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1801): # check between 25th and 26th minutes after stakes have been unfrozen return res = self.get_myself_from_election(config) if not res: From a201d6d66e375c718a3a6c6a4c3f0a4fdd7fed33 Mon Sep 17 00:00:00 2001 From: yungwine Date: Sat, 25 Jan 2025 21:24:17 +0400 Subject: [PATCH 162/206] update stake alert --- modules/alert_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 1e4f5132..521bf0d9 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -337,7 +337,7 @@ def check_stake_returned(self): if not self.ton.using_validator(): return config = self.ton.GetConfig32() - if not (config['endWorkTime'] + FREEZE_PERIOD + 1800 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1801): # check between 25th and 26th minutes after stakes have been unfrozen + if not (config['endWorkTime'] + FREEZE_PERIOD + 1800 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1860): # check between 25th and 26th minutes after stakes have been unfrozen return res = self.get_myself_from_election(config) if not res: From 342fef51b2a9bf41c469860fa80d1fbd23593e54 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 27 Jan 2025 15:44:56 +0400 Subject: [PATCH 163/206] fix chown in restore_backup --- mytonctrl/scripts/restore_backup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index 005ee7d8..9058e91d 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -43,6 +43,8 @@ rm -rf /var/ton-work/db/keyring chown -R $user:$user ${tmp_dir}/mytoncore chown -R $user:$user ${tmp_dir}/keys +chown validator:validator ${tmp_dir}/keys +chown -R validator:validator ${tmp_dir}/db cp -rfp ${tmp_dir}/db /var/ton-work cp -rfp ${tmp_dir}/keys /var/ton-work From 9074f6f4b36d1ba6cf9d63bcd69b3adc20cc9c13 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 27 Jan 2025 15:45:11 +0400 Subject: [PATCH 164/206] abort restore_backup if no backup file --- mytonctrl/scripts/restore_backup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mytonctrl/scripts/restore_backup.sh b/mytonctrl/scripts/restore_backup.sh index 9058e91d..042657d7 100644 --- a/mytonctrl/scripts/restore_backup.sh +++ b/mytonctrl/scripts/restore_backup.sh @@ -16,6 +16,10 @@ do esac done +if [ ! -f "$name" ]; then + echo "Backup file not found, aborting." + exit 1 +fi COLOR='\033[92m' ENDC='\033[0m' From 7528f136a5bc1ff3025bc8096274af082f3dd089 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 28 Jan 2025 11:10:03 +0400 Subject: [PATCH 165/206] rm creation of node config backup on installation --- mytoninstaller/config.py | 11 ----------- mytoninstaller/mytoninstaller.py | 2 -- 2 files changed, 13 deletions(-) diff --git a/mytoninstaller/config.py b/mytoninstaller/config.py index 1be86b79..bf66a127 100644 --- a/mytoninstaller/config.py +++ b/mytoninstaller/config.py @@ -43,17 +43,6 @@ def backup_config(local, config_path): #end define -def BackupVconfig(local): - if local.buffer.only_mtc: - return - local.add_log("Backup validator config file 'config.json' to 'config.json.backup'", "debug") - vconfig_path = local.buffer.vconfig_path - backupPath = vconfig_path + ".backup" - args = ["cp", vconfig_path, backupPath] - subprocess.run(args) -#end define - - def BackupMconfig(local): local.add_log("Backup mytoncore config file 'mytoncore.db' to 'mytoncore.db.backup'", "debug") mconfig_path = local.buffer.mconfig_path diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index cb0dffec..2bc37e49 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -34,7 +34,6 @@ ) from mytoninstaller.config import ( CreateLocalConfig, - BackupVconfig, BackupMconfig, ) @@ -293,7 +292,6 @@ def General(local, console): FirstNodeSettings(local) EnableValidatorConsole(local) EnableLiteServer(local) - BackupVconfig(local) BackupMconfig(local) CreateSymlinks(local) EnableMode(local) From 0340b278cffca3411eae2d3de49c2b968a341efe Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 28 Jan 2025 11:14:36 +0400 Subject: [PATCH 166/206] skip masterchain_out_of_ser and celldb_gc_state metrics if zero values --- modules/prometheus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/prometheus.py b/modules/prometheus.py index 598695ac..203872fa 100644 --- a/modules/prometheus.py +++ b/modules/prometheus.py @@ -44,11 +44,11 @@ def get_validator_status_metrics(self, result: list): result.append(METRICS['master_out_of_sync'].to_format(status.masterchain_out_of_sync)) if status.shardchain_out_of_sync is not None: result.append(METRICS['shard_out_of_sync'].to_format(status.shardchain_out_of_sync)) - if status.masterchain_out_of_ser is not None: + if status.masterchain_out_of_ser is not None and status.stateserializermasterchainseqno != 0: result.append(METRICS['out_of_ser'].to_format(status.masterchain_out_of_ser)) if status.masterchainblock is not None and status.gcmasterchainblock is not None: result.append(METRICS['celldb_gc_block'].to_format(status.masterchainblock - status.gcmasterchainblock)) - if status.gcmasterchainblock is not None and status.last_deleted_mc_state is not None: + if status.gcmasterchainblock is not None and status.last_deleted_mc_state is not None and status.last_deleted_mc_state != 0: result.append(METRICS['celldb_gc_state'].to_format(status.gcmasterchainblock - status.last_deleted_mc_state)) result.append(METRICS['vc_up'].to_format(int(is_working))) From 2cac5794e6bd69dee06ba7d7f827d76cd842aeb2 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 28 Jan 2025 13:55:24 +0400 Subject: [PATCH 167/206] upgrade pip before install in cli mode --- scripts/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install.sh b/scripts/install.sh index 6ba21f15..b6b33647 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -72,6 +72,7 @@ done if [ "${mode}" = "" ]; then # no mode echo "Running cli installer" wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/install.py + python3 -m pip install --upgrade pip pip3 install inquirer==3.4.0 --break-system-packages python3 install.py exit From d153e6f24a0c983a5001334dd89a22a85d9f583f Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:18:51 +0300 Subject: [PATCH 168/206] Improving code readability --- mytoninstaller/node_args.py | 33 +++++++++---------- mytoninstaller/scripts/set_node_argument.py | 36 ++++++++++++++------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/mytoninstaller/node_args.py b/mytoninstaller/node_args.py index 0c5fe3d0..cd8c6e47 100644 --- a/mytoninstaller/node_args.py +++ b/mytoninstaller/node_args.py @@ -2,8 +2,8 @@ def get_validator_service(): path = '/etc/systemd/system/validator.service' - with open(path, 'r') as f: - return f.read() + with open(path, 'r') as file: + return file.read() #end define @@ -14,22 +14,19 @@ def get_node_start_command(): return line.split('=')[1].strip() #end define +def get_node_args(start_command: str = None): + if start_command is None: + start_command = get_node_start_command() + #end if -def get_node_args(command: str = None): - if command is None: - command = get_node_start_command() - result = {} # {key: [value1, value2]} - key = '' - for c in command.split(' ')[1:]: - if c.startswith('--') or c.startswith('-'): - if key: - result[key] = result.get(key, []) + [''] - key = c - elif key: - result[key] = result.get(key, []) + [c] - key = '' - if key: - result[key] = result.get(key, []) + [''] + result = dict() # {key: [value1, value2]} + node_args = start_command.split(' ')[1:] + key = None + for item in node_args: + if item.startswith('-'): + key = item + result[key] = list() + else: + result[key].append(item) return result #end define - diff --git a/mytoninstaller/scripts/set_node_argument.py b/mytoninstaller/scripts/set_node_argument.py index cd241bea..bad88016 100644 --- a/mytoninstaller/scripts/set_node_argument.py +++ b/mytoninstaller/scripts/set_node_argument.py @@ -11,23 +11,35 @@ def set_node_arg(arg_name: str, arg_value: str = ''): """ assert arg_name.startswith('-'), 'arg_name must start with "-" or "--"' service = get_validator_service() - command = get_node_start_command() - if command.split(' ')[0] != '/usr/bin/ton/validator-engine/validator-engine': - raise Exception('Invalid node start command in service file') - if command is None: + start_command = get_node_start_command() + if start_command is None: raise Exception('Cannot find node start command in service file') - args = get_node_args(command) + first_arg = start_command.split(' ')[0] + if first_arg != '/usr/bin/ton/validator-engine/validator-engine': + raise Exception('Invalid node start command in service file') + #end if + + node_args = get_node_args(start_command) if arg_value == '-d': - args.pop(arg_name, None) + node_args.pop(arg_name, None) else: if ' ' in arg_value: - args[arg_name] = arg_value.split() + node_args[arg_name] = arg_value.split() else: - args[arg_name] = [arg_value] - new_command = command.split(' ')[0] + ' ' + ' '.join([f'{k} {v}' for k, vs in args.items() for v in vs]) - new_service = service.replace(command, new_command) - with open('/etc/systemd/system/validator.service', 'w') as f: - f.write(new_service) + node_args[arg_name] = [arg_value] + #end if + + buffer = list() + buffer.append(first_arg) + for key, value_list in node_args.items(): + if len(value_list) == 0: + buffer.append(f"{key}") + for value in value_list: + buffer.append(f"{key} {value}") + new_start_command = ' '.join(buffer) + new_service = service.replace(start_command, new_start_command) + with open('/etc/systemd/system/validator.service', 'w') as file: + file.write(new_service) restart_node() #end define From 92b6c1268fd4d506b5d2716be9bd4a74990ff668 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 31 Jan 2025 10:27:39 +0400 Subject: [PATCH 169/206] fix list_alerts --- modules/alert_bot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 521bf0d9..19d47297 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -218,6 +218,7 @@ def disable_alert(self, args): color_print("disable_alert - {green}OK{endc}") def print_alerts(self, args): + init_alerts() table = [['Name', 'Enabled', 'Last sent']] for alert_name in ALERTS: alert = self.get_alert_from_db(alert_name) From 69f3550fe944db5bab275ec9e8f8e49b663a28fd Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 31 Jan 2025 10:22:03 +0400 Subject: [PATCH 170/206] make celldb_gc_state metric -1 if last_deleted_mc_state is 0 --- modules/prometheus.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/prometheus.py b/modules/prometheus.py index 203872fa..eac13f47 100644 --- a/modules/prometheus.py +++ b/modules/prometheus.py @@ -48,8 +48,11 @@ def get_validator_status_metrics(self, result: list): result.append(METRICS['out_of_ser'].to_format(status.masterchain_out_of_ser)) if status.masterchainblock is not None and status.gcmasterchainblock is not None: result.append(METRICS['celldb_gc_block'].to_format(status.masterchainblock - status.gcmasterchainblock)) - if status.gcmasterchainblock is not None and status.last_deleted_mc_state is not None and status.last_deleted_mc_state != 0: - result.append(METRICS['celldb_gc_state'].to_format(status.gcmasterchainblock - status.last_deleted_mc_state)) + if status.gcmasterchainblock is not None and status.last_deleted_mc_state is not None: + if status.last_deleted_mc_state != 0: + result.append(METRICS['celldb_gc_state'].to_format(status.gcmasterchainblock - status.last_deleted_mc_state)) + else: + result.append(METRICS['celldb_gc_state'].to_format(-1)) result.append(METRICS['vc_up'].to_format(int(is_working))) def get_validator_validation_metrics(self, result: list): From 399b6b00d5e68010d678ccd769c2f1a5a6453c39 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 31 Jan 2025 21:45:33 +0400 Subject: [PATCH 171/206] fix voting alert --- modules/alert_bot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 19d47297..519ed719 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -338,7 +338,7 @@ def check_stake_returned(self): if not self.ton.using_validator(): return config = self.ton.GetConfig32() - if not (config['endWorkTime'] + FREEZE_PERIOD + 1800 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1860): # check between 25th and 26th minutes after stakes have been unfrozen + if not (config['endWorkTime'] + FREEZE_PERIOD + 1800 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1860): # check between 30th and 31st minutes after stakes have been unfrozen return res = self.get_myself_from_election(config) if not res: @@ -357,6 +357,9 @@ def check_voting(self): validator_index = self.ton.GetValidatorIndex() if validator_index == -1: return + config = self.ton.GetConfig34() + if time.time() - config['startWorkTime'] < 600: # less than 10 minutes passed since round start + return need_to_vote = [] offers = self.ton.GetOffers() for offer in offers: From 7198ed1a042aa7034c36838f3a53c399f31f3e02 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 4 Feb 2025 10:13:19 +0400 Subject: [PATCH 172/206] add ExecStartPre = sleep 2 to new installations --- mytoninstaller/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index 8203a15d..88f67a66 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -66,7 +66,7 @@ def FirstNodeSettings(local): # Прописать автозагрузку cpus = psutil.cpu_count() - 1 cmd = f"{validatorAppPath} --threads {cpus} --daemonize --global-config {globalConfigPath} --db {ton_db_dir} --logname {tonLogPath} --archive-ttl {archive_ttl} --verbosity 1" - add2systemd(name="validator", user=vuser, start=cmd) # post="/usr/bin/python3 /usr/src/mytonctrl/mytoncore.py -e \"validator down\"" + add2systemd(name="validator", user=vuser, start=cmd, pre='/bin/sleep 2') # post="/usr/bin/python3 /usr/src/mytonctrl/mytoncore.py -e \"validator down\"" # Получить внешний ip адрес ip = get_own_ip() From 44711d15463f7710d00975cb7236869416d8b5cc Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 4 Feb 2025 10:16:56 +0400 Subject: [PATCH 173/206] fix merge --- mytoninstaller/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index bdec730b..c9b667b3 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -67,14 +67,14 @@ def FirstNodeSettings(local): cpus = psutil.cpu_count() - 1 cmd = f"{validatorAppPath} --threads {cpus} --daemonize --global-config {globalConfigPath} --db {ton_db_dir} --logname {tonLogPath} --archive-ttl {archive_ttl} --verbosity 1" - add2systemd(name="validator", user=vuser, start=cmd, pre='/bin/sleep 2') # post="/usr/bin/python3 /usr/src/mytonctrl/mytoncore.py -e \"validator down\"" - if os.getenv('ADD_SHARD'): add_shard = os.getenv('ADD_SHARD') cmd += f' -M' for shard in add_shard.split(): cmd += f' --add-shard {shard}' + add2systemd(name="validator", user=vuser, start=cmd, pre='/bin/sleep 2') # post="/usr/bin/python3 /usr/src/mytonctrl/mytoncore.py -e \"validator down\"" + # Получить внешний ip адрес ip = get_own_ip() vport = random.randint(2000, 65000) From 3e2330f413c846468a544e7ff91bf504053a8797 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 4 Feb 2025 11:06:40 +0400 Subject: [PATCH 174/206] add shard out of sync in status --- mytonctrl/mytonctrl.py | 6 +++++- mytonctrl/resources/translate.json | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index f0e14c3e..c826dc17 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -728,7 +728,9 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, validatorStatus_color = GetColorStatus(validatorStatus_bool) mytoncoreStatus_text = local.translate("local_status_mytoncore_status").format(mytoncoreStatus_color, mytoncoreUptime_text) validatorStatus_text = local.translate("local_status_validator_status").format(validatorStatus_color, validatorUptime_text) - validator_out_of_sync_text = local.translate("local_status_validator_out_of_sync").format(GetColorInt(validator_status.out_of_sync, 20, logic="less", ending=" s")) + validator_out_of_sync_text = local.translate("local_status_validator_out_of_sync").format(GetColorInt(validator_status.out_of_sync, 20, logic="less")) + master_out_of_sync_text = local.translate("local_status_master_out_of_sync").format(GetColorInt(validator_status.masterchain_out_of_sync, 20, logic="less", ending=" sec")) + shard_out_of_sync_text = local.translate("local_status_shard_out_of_sync").format(GetColorInt(validator_status.shardchain_out_of_sync, 5, logic="less", ending=" blocks")) validator_out_of_ser_text = local.translate("local_status_validator_out_of_ser").format(f'{validator_status.out_of_ser} blocks ago') @@ -776,6 +778,8 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, if not is_node_remote: print(validatorStatus_text) print(validator_out_of_sync_text) + print(master_out_of_sync_text) + print(shard_out_of_sync_text) print(validator_out_of_ser_text) print(dbStatus_text) print(mtcVersion_text) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index ee3f900f..fbbbc1cf 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -324,6 +324,16 @@ "ru": "Рассинхронизация локального валидатора: {0}", "zh_TW": "本地驗證者不同步: {0}" }, + "local_status_master_out_of_sync": { + "en": "Masterchain out of sync: {0}", + "ru": "Рассинхронизация Мастерчейна локального валидатора: {0}", + "zh_TW": "主鏈不同步: {0}" + }, + "local_status_shard_out_of_sync": { + "en": "Shardchain out of sync: {0}", + "ru": "Рассинхронизация Шардчейна локального валидатора: {0}", + "zh_TW": "分片鏈不同步: {0}" + }, "local_status_validator_out_of_ser": { "en": "Local validator last state serialization: {0}", "ru": "Серализация стейта локального валидатора была: {0}", From b9d2e094adf0f94a7612337f3de0d0a6d49bfbbb Mon Sep 17 00:00:00 2001 From: Igroman787 <27614297+igroman787@users.noreply.github.com> Date: Tue, 4 Feb 2025 10:42:24 +0300 Subject: [PATCH 175/206] update submodule --- mypylib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypylib b/mypylib index 049205ee..3508d3aa 160000 --- a/mypylib +++ b/mypylib @@ -1 +1 @@ -Subproject commit 049205eee1650d7d847497d78acba68c5a33cc37 +Subproject commit 3508d3aa07ec3ea1a47690a3e7477b2b8e677c4a From 305117f54afe3269bc2cdd140a9e0541254f7f72 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 10 Feb 2025 09:33:20 +0400 Subject: [PATCH 176/206] fix CreateLocalConfigFile --- mytoninstaller/mytoninstaller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index aaca6ca0..c0827cae 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -195,7 +195,8 @@ def PrintLiteServerConfig(local, args): def CreateLocalConfigFile(local, args): initBlock = GetInitBlock() initBlock_b64 = dict2b64(initBlock) - args = ["python3", "-m", "mytoninstaller", "-u", local.buffer.user, "-e", "clc", "-i", initBlock_b64] + user = local.buffer.user or os.environ.get("USER", "root") + args = ["python3", "-m", "mytoninstaller", "-u", user, "-e", "clc", "-i", initBlock_b64] run_as_root(args) #end define From 22993b83ca1e44e253abaf44d1276d42e76ab800 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 10 Feb 2025 09:39:30 +0400 Subject: [PATCH 177/206] check if tha exists before enabling --- mytoncore/mytoncore.py | 5 +++-- mytoninstaller/utils.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 22d42bd7..bdc89aea 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -33,6 +33,8 @@ Dict, int2ip ) +from mytoninstaller.utils import enable_tha + class MyTonCore(): def __init__(self, local): @@ -3079,8 +3081,7 @@ def check_enable_mode(self, name): raise Exception(f'Cannot enable validator mode while liteserver mode is enabled. ' f'Use `disable_mode liteserver` first.') if name == 'liquid-staking': - from mytoninstaller.settings import enable_ton_http_api - enable_ton_http_api(self.local) + enable_tha(self.local) def enable_mode(self, name): if name not in MODES: diff --git a/mytoninstaller/utils.py b/mytoninstaller/utils.py index a2cd817f..4c2fdf0f 100644 --- a/mytoninstaller/utils.py +++ b/mytoninstaller/utils.py @@ -2,6 +2,8 @@ import json import time import subprocess + +import requests from nacl.signing import SigningKey @@ -50,3 +52,25 @@ def get_ed25519_pubkey(privkey): pubkey = privkey_obj.verify_key.encode() return pubkey #end define + + +def tha_exists(): + try: + resp = requests.get('http://127.0.0.1:8801/healthcheck', timeout=3) + except: + return False + if resp.status_code == 200 and resp.text == '"OK"': + return True + return False + + +def enable_tha(local): + try: + if not tha_exists(): + local.add_log("Enabling TON HTTP API", "debug") + from mytoninstaller.settings import enable_ton_http_api + enable_ton_http_api(local) + local.add_log("Enabled TON HTTP API", "debug") + except Exception as e: + local.add_log(f"Error in enable_tha: {e}", "warning") + pass From 8a4661023b2706c94063e6ad1ccd301c37c07529 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 6 Feb 2025 13:56:27 +0400 Subject: [PATCH 178/206] add skip-create-backup flag to restore_backup --- modules/backups.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/backups.py b/modules/backups.py index 0458c298..8ea0b2fa 100644 --- a/modules/backups.py +++ b/modules/backups.py @@ -56,8 +56,8 @@ def run_restore_backup(args): return run_as_root(["bash", restore_script_path] + args) def restore_backup(self, args): - if len(args) == 0 or len(args) > 2: - color_print("{red}Bad args. Usage:{endc} restore_backup [-y]") + if len(args) == 0 or len(args) > 3: + color_print("{red}Bad args. Usage:{endc} restore_backup [-y] [--skip-create-backup]") return if '-y' not in args: res = input( @@ -67,11 +67,14 @@ def restore_backup(self, args): return else: args.pop(args.index('-y')) - print('Before proceeding, mtc will create a backup of current configuration.') - try: - self.create_backup([]) - except: - color_print("{red}Could not create backup{endc}") + if '--skip-create-backup' in args: + args.pop(args.index('--skip-create-backup')) + else: + print('Before proceeding, mtc will create a backup of current configuration.') + try: + self.create_backup([]) + except: + color_print("{red}Could not create backup{endc}") ip = str(ip2int(get_own_ip())) command_args = ["-m", self.ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip] From 0c398a1a2365d641d387df6d96b7265f81f1a3d1 Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 11 Feb 2025 11:32:36 +0400 Subject: [PATCH 179/206] update alert message format --- modules/alert_bot.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 45a88a98..318d604a 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -115,6 +115,7 @@ def __init__(self, ton, local, *args, **kwargs): self.inited = False self.hostname = None self.ip = None + self.adnl = None self.token = None self.chat_id = None self.last_db_check = 0 @@ -141,16 +142,19 @@ def send_alert(self, alert_name: str, *args, **kwargs): alert = ALERTS.get(alert_name) if alert is None: raise Exception(f"Alert {alert_name} not found") - text = f''' -❗️ MyTonCtrl Alert {alert_name} ❗️ + alert_name_readable = alert_name.replace('_', ' ').title() + text = '🆘' if alert.severity != 'info' else '' + text += f''' +️ Node {self.hostname}: {alert_name_readable} + +{alert.text.format(*args, **kwargs)} Hostname: {self.hostname} Node IP: {self.ip} +ADNL: {self.adnl} Time: {time_} ({int(time.time())}) +Alert name: {alert_name} Severity: {alert.severity} - -Alert text: -
{alert.text.format(*args, **kwargs)}
''' if time.time() - last_sent > alert.timeout: self.send_message(text, alert.severity == "info") # send info alerts without sound @@ -174,6 +178,8 @@ def init(self): from modules.validator import ValidatorModule self.validator_module = ValidatorModule(self.ton, self.local) self.hostname = get_hostname() + adnl = self.ton.GetAdnlAddr() + self.adnl = adnl[:4] + '...' + adnl[-4:] self.ip = self.ton.get_node_ip() self.set_global_vars() init_alerts() From e2ed02d41ea45f9aa6be149ef62a240b664578fb Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 11 Feb 2025 22:36:24 +0400 Subject: [PATCH 180/206] add welcome message --- modules/alert_bot.py | 63 ++++++++++++++++++++++++++++++++++++------ mytonctrl/mytonctrl.py | 6 +++- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 318d604a..5799f185 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -11,6 +11,7 @@ @dataclasses.dataclass class Alert: severity: str + description: str text: str timeout: int @@ -29,11 +30,13 @@ def init_alerts(): ALERTS = { "low_wallet_balance": Alert( "low", - "Validator wallet {wallet} balance is low: {balance} TON.", + "Validator's wallet balance is low", + "Validator's wallet {wallet} balance is less than 10 TON: {balance} TON.", 18 * HOUR ), "db_usage_80": Alert( "high", + "Node's db usage is more than 80%", """TON DB usage > 80%. Clean the TON database: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming or (and) set node\'s archive ttl to lower value.""", @@ -41,6 +44,7 @@ def init_alerts(): ), "db_usage_95": Alert( "critical", + "Node's db usage is more than 95%", """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming or (and) set node\'s archive ttl to lower value.""", @@ -48,56 +52,67 @@ def init_alerts(): ), "low_efficiency": Alert( "high", - """Validator efficiency is low: {efficiency}%.""", + "Validator had low efficiency in the validation round", + """Validator efficiency is less than 90%: {efficiency}%.""", VALIDATION_PERIOD // 3 ), "out_of_sync": Alert( "critical", - "Node is out of sync on {sync} sec.", + "Validator had low efficiency in the round", + "Node is out of sync on more than 20 sec: {sync} sec.", 300 ), "service_down": Alert( "critical", + "Node is not running (service is down)", "validator.service is down.", 300 ), "adnl_connection_failed": Alert( "high", + "Node is not answering to ADNL connection", "ADNL connection to node failed", 3 * HOUR ), "zero_block_created": Alert( "critical", + "Validator has not created any blocks in the last few hours", "Validator has not created any blocks in the last {hours} hours.", VALIDATION_PERIOD // 3 ), "validator_slashed": Alert( "high", + "Validator has been slashed in the previous validation round", "Validator has been slashed in previous round for {amount} TON", FREEZE_PERIOD ), "stake_not_accepted": Alert( "high", "Validator's stake has not been accepted", + "Validator's stake has not been accepted", ELECTIONS_START_BEFORE ), "stake_accepted": Alert( "info", + "Validator's stake has been accepted (info alert with no sound)", "Validator's stake {stake} TON has been accepted", ELECTIONS_START_BEFORE ), "stake_returned": Alert( "info", + "Validator's stake has been returned (info alert with no sound)", "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", 60 ), "stake_not_returned": Alert( "high", + "Validator's stake has not been returned", "Validator's stake has not been returned on address {address}.", 60 ), "voting": Alert( "high", + "There is an active network proposal that has many votes but is not voted by the validator", "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.", VALIDATION_PERIOD ), @@ -120,14 +135,14 @@ def __init__(self, ton, local, *args, **kwargs): self.chat_id = None self.last_db_check = 0 - def send_message(self, text: str, silent: bool = False): + def send_message(self, text: str, silent: bool = False, disable_web_page_preview: bool = False): if self.token is None: raise Exception("send_message error: token is not initialized") if self.chat_id is None: raise Exception("send_message error: chat_id is not initialized") request_url = f"https://api.telegram.org/bot{self.token}/sendMessage" - data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML', 'disable_notification': silent} - response = requests.post(request_url, data=data, timeout=3) + data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML', 'disable_notification': silent, 'link_preview_options': {'is_disabled': disable_web_page_preview}} + response = requests.post(request_url, json=data, timeout=3) if response.status_code != 200: raise Exception(f"send_message error: {response.text}") response = response.json() @@ -144,8 +159,7 @@ def send_alert(self, alert_name: str, *args, **kwargs): raise Exception(f"Alert {alert_name} not found") alert_name_readable = alert_name.replace('_', ' ').title() text = '🆘' if alert.severity != 'info' else '' - text += f''' -️ Node {self.hostname}: {alert_name_readable} + text += f''' Node {self.hostname}: {alert_name_readable} {alert.text.format(*args, **kwargs)} @@ -236,6 +250,39 @@ def test_alert(self, args): self.init() self.send_message('Test alert') + def send_welcome_message(self): + message = """ +👋 Hello, this is alert bot. + +I don't process any commands, I only send notifications. + +Current notifications enabled: + +""" + for alert in ALERTS.values(): + message += f"- {alert.description}\n" + + message += """ +If you want, you can disable some notifications in mytonctrl by the instruction . + +Full bot documentation here. +""" + self.send_message(text=message, disable_web_page_preview=True) + + def on_set_chat_id(self, chat_id): + self.token = self.ton.local.db.get("BotToken") + if self.token is None: + raise Exception("BotToken is not set") + self.chat_id = chat_id + init_alerts() + try: + self.send_welcome_message() + return True + except Exception as e: + self.local.add_log(f"Error while sending welcome message: {e}", "error") + self.local.add_log(f"If you want the bot to write to a multi-person chat group, make sure the bot is added to that chat group and has admin rights. If it is not - do it and run the command `set ChatId ` again.", "info") + return False + def check_db_usage(self): if time.time() - self.last_db_check < 600: return diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index f0e14c3e..ffb7252b 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -879,7 +879,7 @@ def GetSettings(ton, args): print(json.dumps(result, indent=2)) #end define -def SetSettings(ton, args): +def SetSettings(local, ton, args): try: name = args[0] value = args[1] @@ -891,6 +891,10 @@ def SetSettings(ton, args): color_print(f"{{red}} Error: set {name} ... is deprecated and does not work {{endc}}." f"\nInstead, use {{bold}}enable_mode {mode_name}{{endc}}") return + if name == 'ChatId': + from modules.alert_bot import AlertBotModule + if not AlertBotModule(ton, local).on_set_chat_id(value): + return force = False if len(args) > 2: if args[2] == "--force": From 1f18ee255c0015f4b963c033441f9f486b2eec57 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 12 Feb 2025 15:58:19 +0400 Subject: [PATCH 181/206] fix changing chat id for bot --- modules/alert_bot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 5799f185..ef7049ec 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -424,7 +424,8 @@ def check_voting(self): def check_status(self): if not self.ton.using_alert_bot(): return - if not self.inited: + + if not self.inited or self.token != self.ton.local.db.get("BotToken") or self.chat_id != self.ton.local.db.get("ChatId"): self.init() self.local.try_function(self.check_db_usage) From 6fdba7ad2c3235d862707076ad03544d1bc1e3d3 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 12 Feb 2025 17:12:33 +0400 Subject: [PATCH 182/206] alert bot ux improves --- modules/alert_bot.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index ef7049ec..04fa34bf 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -30,7 +30,7 @@ def init_alerts(): ALERTS = { "low_wallet_balance": Alert( "low", - "Validator's wallet balance is low", + "Validator's wallet balance is less than 10 TON", "Validator's wallet {wallet} balance is less than 10 TON: {balance} TON.", 18 * HOUR ), @@ -52,13 +52,13 @@ def init_alerts(): ), "low_efficiency": Alert( "high", - "Validator had low efficiency in the validation round", + "Validator had efficiency less than 90% in the validation round", """Validator efficiency is less than 90%: {efficiency}%.""", VALIDATION_PERIOD // 3 ), "out_of_sync": Alert( "critical", - "Validator had low efficiency in the round", + "Node is out of sync on more than 20 sec", "Node is out of sync on more than 20 sec: {sync} sec.", 300 ), @@ -76,7 +76,7 @@ def init_alerts(): ), "zero_block_created": Alert( "critical", - "Validator has not created any blocks in the last few hours", + f"Validator has not created any blocks in the {int(VALIDATION_PERIOD // 3 // 3600)} hours", "Validator has not created any blocks in the last {hours} hours.", VALIDATION_PERIOD // 3 ), @@ -101,18 +101,18 @@ def init_alerts(): "stake_returned": Alert( "info", "Validator's stake has been returned (info alert with no sound)", - "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", + "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", 60 ), "stake_not_returned": Alert( "high", "Validator's stake has not been returned", - "Validator's stake has not been returned on address {address}.", + "Validator's stake has not been returned on address {address}.", 60 ), "voting": Alert( "high", - "There is an active network proposal that has many votes but is not voted by the validator", + "There is an active network proposal that has many votes (more than 50% of required) but is not voted by the validator", "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.", VALIDATION_PERIOD ), @@ -166,6 +166,7 @@ def send_alert(self, alert_name: str, *args, **kwargs): Hostname: {self.hostname} Node IP: {self.ip} ADNL: {self.adnl} +Wallet: {self.wallet} Time: {time_} ({int(time.time())}) Alert name: {alert_name} Severity: {alert.severity} @@ -193,7 +194,8 @@ def init(self): self.validator_module = ValidatorModule(self.ton, self.local) self.hostname = get_hostname() adnl = self.ton.GetAdnlAddr() - self.adnl = adnl[:4] + '...' + adnl[-4:] + self.adnl = adnl + self.wallet = self.ton.GetValidatorWallet().addrB64 self.ip = self.ton.get_node_ip() self.set_global_vars() init_alerts() @@ -251,8 +253,8 @@ def test_alert(self, args): self.send_message('Test alert') def send_welcome_message(self): - message = """ -👋 Hello, this is alert bot. + message = f""" +This is alert bot. You have connected validator with ADNL {self.ton.GetAdnlAddr()}. I don't process any commands, I only send notifications. @@ -263,9 +265,9 @@ def send_welcome_message(self): message += f"- {alert.description}\n" message += """ -If you want, you can disable some notifications in mytonctrl by the instruction . +If you want, you can disable some notifications in mytonctrl by the instruction. -Full bot documentation here. +Full bot documentation here. """ self.send_message(text=message, disable_web_page_preview=True) @@ -280,7 +282,7 @@ def on_set_chat_id(self, chat_id): return True except Exception as e: self.local.add_log(f"Error while sending welcome message: {e}", "error") - self.local.add_log(f"If you want the bot to write to a multi-person chat group, make sure the bot is added to that chat group and has admin rights. If it is not - do it and run the command `set ChatId ` again.", "info") + self.local.add_log(f"If you want the bot to write to a multi-person chat group, make sure the bot is added to that chat group. If it is not - do it and run the command `set ChatId ` again.", "info") return False def check_db_usage(self): From 371a4a477e810c383d4194f9ce628ae501471e72 Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 13 Feb 2025 13:22:47 +0400 Subject: [PATCH 183/206] refactor enable tha --- mytoncore/mytoncore.py | 5 ++--- mytoninstaller/settings.py | 25 ++++++++++++++++++++++--- mytoninstaller/utils.py | 24 ------------------------ 3 files changed, 24 insertions(+), 30 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index bdc89aea..22d42bd7 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -33,8 +33,6 @@ Dict, int2ip ) -from mytoninstaller.utils import enable_tha - class MyTonCore(): def __init__(self, local): @@ -3081,7 +3079,8 @@ def check_enable_mode(self, name): raise Exception(f'Cannot enable validator mode while liteserver mode is enabled. ' f'Use `disable_mode liteserver` first.') if name == 'liquid-staking': - enable_tha(self.local) + from mytoninstaller.settings import enable_ton_http_api + enable_ton_http_api(self.local) def enable_mode(self, name): if name not in MODES: diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index c9b667b3..d81f572d 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -485,17 +485,36 @@ def EnableJsonRpc(local): color_print(text) #end define +def tha_exists(): + try: + resp = requests.get('http://127.0.0.1:8801/healthcheck', timeout=3) + except: + return False + if resp.status_code == 200 and resp.text == '"OK"': + return True + return False +#end define + def enable_ton_http_api(local): - local.add_log("start EnableTonHttpApi function", "debug") + try: + if not tha_exists(): + enable_ton_http_api(local) + except Exception as e: + local.add_log(f"Error in enable_ton_http_api: {e}", "warning") + pass +#end define + +def do_enable_ton_http_api(local): + local.add_log("start do_enable_ton_http_api function", "debug") if not os.path.exists('/usr/bin/ton/local.config.json'): from mytoninstaller.mytoninstaller import CreateLocalConfigFile CreateLocalConfigFile(local, []) ton_http_api_installer_path = pkg_resources.resource_filename('mytoninstaller.scripts', 'ton_http_api_installer.sh') exit_code = run_as_root(["bash", ton_http_api_installer_path]) if exit_code == 0: - text = "EnableTonHttpApi - {green}OK{endc}" + text = "do_enable_ton_http_api - {green}OK{endc}" else: - text = "EnableTonHttpApi - {red}Error{endc}" + text = "do_enable_ton_http_api - {red}Error{endc}" color_print(text) #end define diff --git a/mytoninstaller/utils.py b/mytoninstaller/utils.py index 4c2fdf0f..a2cd817f 100644 --- a/mytoninstaller/utils.py +++ b/mytoninstaller/utils.py @@ -2,8 +2,6 @@ import json import time import subprocess - -import requests from nacl.signing import SigningKey @@ -52,25 +50,3 @@ def get_ed25519_pubkey(privkey): pubkey = privkey_obj.verify_key.encode() return pubkey #end define - - -def tha_exists(): - try: - resp = requests.get('http://127.0.0.1:8801/healthcheck', timeout=3) - except: - return False - if resp.status_code == 200 and resp.text == '"OK"': - return True - return False - - -def enable_tha(local): - try: - if not tha_exists(): - local.add_log("Enabling TON HTTP API", "debug") - from mytoninstaller.settings import enable_ton_http_api - enable_ton_http_api(local) - local.add_log("Enabled TON HTTP API", "debug") - except Exception as e: - local.add_log(f"Error in enable_tha: {e}", "warning") - pass From faf34895a8192edf6081747508bb3123d727026c Mon Sep 17 00:00:00 2001 From: yungwine Date: Thu, 13 Feb 2025 13:27:52 +0400 Subject: [PATCH 184/206] fix refactor enable tha --- mytoninstaller/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index d81f572d..f8452dba 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -498,7 +498,7 @@ def tha_exists(): def enable_ton_http_api(local): try: if not tha_exists(): - enable_ton_http_api(local) + do_enable_ton_http_api(local) except Exception as e: local.add_log(f"Error in enable_ton_http_api: {e}", "warning") pass From 02bfb3b580fcc30338a5158b1901218e4b83390a Mon Sep 17 00:00:00 2001 From: yungwine Date: Sun, 16 Feb 2025 12:26:49 +0400 Subject: [PATCH 185/206] create mtc db backup more often --- mytoncore/mytoncore.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 22d42bd7..e9f2412f 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -117,10 +117,17 @@ def CheckConfigFile(self, fift, liteClient): subprocess.run(args) self.dbFile = mconfig_path self.Refresh() - elif os.path.isfile(backup_path) == False: + elif not os.path.isfile(backup_path) or time.time() - os.path.getmtime(backup_path) > 3600: self.local.add_log("Create backup config file", "info") - args = ["cp", mconfig_path, backup_path] - subprocess.run(args) + backup_tmp_path = backup_path + '.tmp' + subprocess.run(["cp", mconfig_path, backup_tmp_path]) + try: + with open(backup_tmp_path, "r") as file: + json.load(file) + os.rename(backup_tmp_path, backup_path) # atomic opetation + except: + self.local.add_log("Could not update backup, backup_tmp file is broken", "warning") + os.remove(backup_tmp_path) #end define def GetVarFromWorkerOutput(self, text, search): From c95a177181b8ee6dd3e608941e5f1f6257edbff2 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 17 Feb 2025 13:23:37 +0400 Subject: [PATCH 186/206] create mtc db backup after every set --- mytoncore/mytoncore.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e9f2412f..63c571ed 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -118,18 +118,23 @@ def CheckConfigFile(self, fift, liteClient): self.dbFile = mconfig_path self.Refresh() elif not os.path.isfile(backup_path) or time.time() - os.path.getmtime(backup_path) > 3600: - self.local.add_log("Create backup config file", "info") - backup_tmp_path = backup_path + '.tmp' - subprocess.run(["cp", mconfig_path, backup_tmp_path]) - try: - with open(backup_tmp_path, "r") as file: - json.load(file) - os.rename(backup_tmp_path, backup_path) # atomic opetation - except: - self.local.add_log("Could not update backup, backup_tmp file is broken", "warning") - os.remove(backup_tmp_path) + self.local.try_function(self.create_self_db_backup) #end define + def create_self_db_backup(self): + self.local.add_log("Create backup config file", "info") + mconfig_path = self.local.buffer.db_path + backup_path = mconfig_path + ".backup" + backup_tmp_path = backup_path + '.tmp' + subprocess.run(["cp", mconfig_path, backup_tmp_path]) + try: + with open(backup_tmp_path, "r") as file: + json.load(file) + os.rename(backup_tmp_path, backup_path) # atomic opetation + except: + self.local.add_log("Could not update backup, backup_tmp file is broken", "warning") + os.remove(backup_tmp_path) + def GetVarFromWorkerOutput(self, text, search): if ':' not in search: search += ':' @@ -3044,6 +3049,7 @@ def SetSettings(self, name, data): except: pass self.local.db[name] = data self.local.save() + self.create_self_db_backup() #end define def migrate_to_modes(self): From 028e67e8a970456d3aa8e10a2ce48cb2ac13b13b Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 21 Feb 2025 12:01:00 +0400 Subject: [PATCH 187/206] add new validator status fields parsing, add initial sync status and warning --- mytoncore/mytoncore.py | 10 ++++++++++ mytonctrl/mytonctrl.py | 10 ++++++++++ mytonctrl/resources/translate.json | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 22d42bd7..e9689a64 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -764,6 +764,13 @@ def GetShardsNumber(self, block=None): return shardsNum #end define + def parse_stats_from_vc(self, output: str, result: dict): + for line in output.split('\n'): + if len(line.split('\t\t\t')) == 2: + name, value = line.split('\t\t\t') # https://github.com/ton-blockchain/ton/blob/master/validator-engine-console/validator-engine-console-query.cpp#L648 + if name not in result: + result[name] = value + def GetValidatorStatus(self): # Get buffer bname = "validator_status" @@ -797,10 +804,13 @@ def GetValidatorStatus(self): status.out_of_sync = status.masterchain_out_of_sync if status.masterchain_out_of_sync > status.shardchain_out_of_sync else status.shardchain_out_of_sync status.out_of_ser = status.masterchain_out_of_ser status.last_deleted_mc_state = int(parse(result, "last_deleted_mc_state", '\n')) + self.local.try_function(self.parse_stats_from_vc, args=[result, status]) except Exception as ex: self.local.add_log(f"GetValidatorStatus warning: {ex}", "warning") status.is_working = False + self.local.try_function(self.parse_stats_from_vc, args=[result, status]) #end try + status.initial_sync = status.get("process.initial_sync") # old vars status.outOfSync = status.out_of_sync diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index b4ec1fb1..577e0911 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -440,6 +440,9 @@ def check_disk_usage(local, ton): def check_sync(local, ton): validator_status = ton.GetValidatorStatus() + if validator_status.initial_sync: + print_warning(local, "initial_sync_warning") + return if not validator_status.is_working or validator_status.out_of_sync >= 20: print_warning(local, "sync_warning") #end define @@ -728,6 +731,11 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, validatorStatus_color = GetColorStatus(validatorStatus_bool) mytoncoreStatus_text = local.translate("local_status_mytoncore_status").format(mytoncoreStatus_color, mytoncoreUptime_text) validatorStatus_text = local.translate("local_status_validator_status").format(validatorStatus_color, validatorUptime_text) + + validator_initial_sync_text = '' + + if validator_status.initial_sync: + validator_initial_sync_text = local.translate("local_status_validator_initial_sync").format(validator_status['process.initial_sync']) validator_out_of_sync_text = local.translate("local_status_validator_out_of_sync").format(GetColorInt(validator_status.out_of_sync, 20, logic="less")) master_out_of_sync_text = local.translate("local_status_master_out_of_sync").format(GetColorInt(validator_status.masterchain_out_of_sync, 20, logic="less", ending=" sec")) shard_out_of_sync_text = local.translate("local_status_shard_out_of_sync").format(GetColorInt(validator_status.shardchain_out_of_sync, 5, logic="less", ending=" blocks")) @@ -777,6 +785,8 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, print(mytoncoreStatus_text) if not is_node_remote: print(validatorStatus_text) + if validator_status.initial_sync: + print(validator_initial_sync_text) print(validator_out_of_sync_text) print(master_out_of_sync_text) print(shard_out_of_sync_text) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index fbbbc1cf..bb8bf284 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -319,6 +319,11 @@ "ru": "Статус локального валидатора: {0}, {1}", "zh_TW": "本地驗證者狀態: {0}, {1}" }, + "local_status_validator_initial_sync": { + "en": "Local validator initial sync status: {0}", + "ru": "Статус начальной синхронизации локального валидатора: {0}", + "zh_TW": "本地驗證者初始同步狀態: {0}" + }, "local_status_validator_out_of_sync": { "en": "Local validator out of sync: {0}", "ru": "Рассинхронизация локального валидатора: {0}", @@ -439,6 +444,11 @@ "ru": "{red}Убедитесь, что вы подписаны на канал валидаторов TON в Telegram: https://t.me/tonstatus {endc}\nЧтобы отключить это предупреждение, используйте команду `set subscribe_tg_channel true`", "zh_TW": "{red}確保您已訂閱了 Telegram 上的 TON 驗證者頻道: https://t.me/tonstatus {endc}\n要禁用此警告，請使用命令 `set subscribe_tg_channel true`" }, + "initial_sync_warning": { + "en": "{red}Initial Node sync is not completed. The displayed status is incomplete. {endc}", + "ru": "{red}Начальная синхронизация ноды не завершена. Отображаемый статус не полный. {endc}", + "zh_TW": "{red}初始節點同步未完成。顯示的狀態不完整。 {endc}" + }, "sync_warning": { "en": "{red}Node is out of sync. The displayed status is incomplete. {endc}", "ru": "{red}Нода не синхронизирована с сетью. Отображаемый статус не полный. {endc}", From c4ac040ed7208adf6c3d73b704f4babed5aa0840 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 24 Feb 2025 11:14:26 +0400 Subject: [PATCH 188/206] add initial sync to mtc --- mytoncore/functions.py | 14 ++++++++++++++ mytoncore/mytoncore.py | 11 ++++++++++- mytonctrl/mytonctrl.py | 23 +++++++++++++++-------- mytoninstaller/settings.py | 2 ++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/mytoncore/functions.py b/mytoncore/functions.py index a8c08d76..66080db6 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -544,6 +544,17 @@ def ScanLiteServers(local, ton): # end define +def check_initial_sync(local, ton): + if not ton.in_initial_sync(): + return + validator_status = ton.GetValidatorStatus() + if validator_status.initial_sync: + return + if validator_status.out_of_sync < 20: + ton.set_initial_sync_off() + return + + def General(local): local.add_log("start General function", "debug") ton = MyTonCore(local) @@ -579,6 +590,9 @@ def General(local): from modules.prometheus import PrometheusModule local.start_cycle(PrometheusModule(ton, local).push_metrics, sec=30, args=()) + if ton.in_initial_sync(): + local.start_cycle(check_initial_sync, sec=120, args=(local, ton)) + thr_sleep() # end define diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e9689a64..d78b46f2 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -781,6 +781,7 @@ def GetValidatorStatus(self): self.local.add_log("start GetValidatorStatus function", "debug") status = Dict() + result = None try: # Parse status.is_working = True @@ -808,7 +809,8 @@ def GetValidatorStatus(self): except Exception as ex: self.local.add_log(f"GetValidatorStatus warning: {ex}", "warning") status.is_working = False - self.local.try_function(self.parse_stats_from_vc, args=[result, status]) + if result is not None: + self.local.try_function(self.parse_stats_from_vc, args=[result, status]) #end try status.initial_sync = status.get("process.initial_sync") @@ -3137,6 +3139,13 @@ def using_alert_bot(self): def using_prometheus(self): return self.get_mode_value('prometheus') + def in_initial_sync(self): + return self.local.db.get('initialSync', False) + + def set_initial_sync_off(self): + self.local.db.pop('initialSync', None) + self.local.save() + def Tlb2Json(self, text): # Заменить скобки start = 0 diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 577e0911..63f91c9e 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -440,7 +440,7 @@ def check_disk_usage(local, ton): def check_sync(local, ton): validator_status = ton.GetValidatorStatus() - if validator_status.initial_sync: + if validator_status.initial_sync or ton.in_initial_sync(): print_warning(local, "initial_sync_warning") return if not validator_status.is_working or validator_status.out_of_sync >= 20: @@ -733,12 +733,18 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, validatorStatus_text = local.translate("local_status_validator_status").format(validatorStatus_color, validatorUptime_text) validator_initial_sync_text = '' + validator_out_of_sync_text = '' if validator_status.initial_sync: validator_initial_sync_text = local.translate("local_status_validator_initial_sync").format(validator_status['process.initial_sync']) - validator_out_of_sync_text = local.translate("local_status_validator_out_of_sync").format(GetColorInt(validator_status.out_of_sync, 20, logic="less")) - master_out_of_sync_text = local.translate("local_status_master_out_of_sync").format(GetColorInt(validator_status.masterchain_out_of_sync, 20, logic="less", ending=" sec")) - shard_out_of_sync_text = local.translate("local_status_shard_out_of_sync").format(GetColorInt(validator_status.shardchain_out_of_sync, 5, logic="less", ending=" blocks")) + elif ton.in_initial_sync(): # states have been downloaded, now downloading blocks + validator_initial_sync_text = local.translate("local_status_validator_initial_sync").format( + f'Syncing blocks, last known block was {validator_status.out_of_sync} s ago' + ) + else: + validator_out_of_sync_text = local.translate("local_status_validator_out_of_sync").format(GetColorInt(validator_status.out_of_sync, 20, logic="less")) + master_out_of_sync_text = local.translate("local_status_master_out_of_sync").format(GetColorInt(validator_status.masterchain_out_of_sync, 20, logic="less", ending=" sec")) + shard_out_of_sync_text = local.translate("local_status_shard_out_of_sync").format(GetColorInt(validator_status.shardchain_out_of_sync, 5, logic="less", ending=" blocks")) validator_out_of_ser_text = local.translate("local_status_validator_out_of_ser").format(f'{validator_status.out_of_ser} blocks ago') @@ -785,11 +791,12 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, print(mytoncoreStatus_text) if not is_node_remote: print(validatorStatus_text) - if validator_status.initial_sync: + if validator_initial_sync_text: print(validator_initial_sync_text) - print(validator_out_of_sync_text) - print(master_out_of_sync_text) - print(shard_out_of_sync_text) + if validator_out_of_sync_text: + print(validator_out_of_sync_text) + print(master_out_of_sync_text) + print(shard_out_of_sync_text) print(validator_out_of_ser_text) print(dbStatus_text) print(mtcVersion_text) diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index c9b667b3..ccbe6605 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -206,6 +206,8 @@ def FirstMytoncoreSettings(local): # Telemetry mconfig.sendTelemetry = local.buffer.telemetry + mconfig.initialSync = True + # Записать настройки в файл SetConfig(path=mconfig_path, data=mconfig) From f1ea2283abc3733f2caebb766fabbe34001a1015 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 24 Feb 2025 11:14:36 +0400 Subject: [PATCH 189/206] add initial sync alert do not send some alerts if initial sync --- modules/alert_bot.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 04fa34bf..0107486b 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -116,6 +116,12 @@ def init_alerts(): "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.", VALIDATION_PERIOD ), + "initial_sync_completed": Alert( + "info", + "Initial sync has been completed (info alert with no sound)", + "Node initial sync has been completed", + 0 + ) } @@ -134,6 +140,7 @@ def __init__(self, ton, local, *args, **kwargs): self.token = None self.chat_id = None self.last_db_check = 0 + self.initial_sync = None def send_message(self, text: str, silent: bool = False, disable_web_page_preview: bool = False): if self.token is None: @@ -198,6 +205,7 @@ def init(self): self.wallet = self.ton.GetValidatorWallet().addrB64 self.ip = self.ton.get_node_ip() self.set_global_vars() + self.initial_sync = self.ton.in_initial_sync() init_alerts() self.inited = True @@ -323,12 +331,12 @@ def check_efficiency(self): def check_validator_working(self): validator_status = self.ton.GetValidatorStatus() - if not validator_status.is_working: + if not self.initial_sync and not validator_status.is_working: self.send_alert("service_down") def check_sync(self): validator_status = self.ton.GetValidatorStatus() - if validator_status.is_working and validator_status.out_of_sync >= 20: + if not self.initial_sync and validator_status.is_working and validator_status.out_of_sync >= 20: self.send_alert("out_of_sync", sync=validator_status.out_of_sync) def check_zero_blocks_created(self): @@ -423,6 +431,13 @@ def check_voting(self): if need_to_vote: self.send_alert("voting", hashes=' '.join(need_to_vote)) + def check_initial_sync(self): + if not self.initial_sync: + return + if not self.ton.in_initial_sync(): + self.initial_sync = False + self.send_alert("initial_sync_completed") + def check_status(self): if not self.ton.using_alert_bot(): return @@ -441,6 +456,7 @@ def check_status(self): self.local.try_function(self.check_stake_sent) self.local.try_function(self.check_stake_returned) self.local.try_function(self.check_voting) + self.local.try_function(self.check_initial_sync) def add_console_commands(self, console): console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd")) From daa1b9be877a9b743364a137b99b249f0c4710de Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 25 Feb 2025 17:09:34 +0400 Subject: [PATCH 190/206] update setting initial sync --- mytoninstaller/mytoninstaller.py | 3 ++- mytoninstaller/settings.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py index aaca6ca0..cf3ce534 100644 --- a/mytoninstaller/mytoninstaller.py +++ b/mytoninstaller/mytoninstaller.py @@ -30,7 +30,7 @@ enable_ls_proxy, enable_ton_storage, enable_ton_storage_provider, - EnableMode, ConfigureFromBackup, ConfigureOnlyNode + EnableMode, ConfigureFromBackup, ConfigureOnlyNode, SetInitialSync ) from mytoninstaller.config import ( CreateLocalConfig, @@ -299,6 +299,7 @@ def General(local, console): EnableMode(local) ConfigureFromBackup(local) ConfigureOnlyNode(local) + SetInitialSync(local) #end define diff --git a/mytoninstaller/settings.py b/mytoninstaller/settings.py index ccbe6605..62b2f088 100644 --- a/mytoninstaller/settings.py +++ b/mytoninstaller/settings.py @@ -206,8 +206,6 @@ def FirstMytoncoreSettings(local): # Telemetry mconfig.sendTelemetry = local.buffer.telemetry - mconfig.initialSync = True - # Записать настройки в файл SetConfig(path=mconfig_path, data=mconfig) @@ -1000,3 +998,13 @@ def ConfigureOnlyNode(local): SetConfig(path=mconfig_path, data=mconfig) start_service(local, 'mytoncore') + + +def SetInitialSync(local): + mconfig_path = local.buffer.mconfig_path + + mconfig = GetConfig(path=mconfig_path) + mconfig.initialSync = True + SetConfig(path=mconfig_path, data=mconfig) + + start_service(local, 'mytoncore') From b9c4199ec693850d8f9f83138211353afa488715 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 26 Feb 2025 11:08:10 +0400 Subject: [PATCH 191/206] do not send wallet address in alerts if not validator --- modules/alert_bot.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 04fa34bf..991118af 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -31,7 +31,7 @@ def init_alerts(): "low_wallet_balance": Alert( "low", "Validator's wallet balance is less than 10 TON", - "Validator's wallet {wallet} balance is less than 10 TON: {balance} TON.", + "Validator's wallet {wallet} balance is less than 10 TON: {balance} TON.", 18 * HOUR ), "db_usage_80": Alert( @@ -165,8 +165,12 @@ def send_alert(self, alert_name: str, *args, **kwargs): Hostname: {self.hostname} Node IP: {self.ip} -ADNL: {self.adnl} -Wallet: {self.wallet} +ADNL: {self.adnl}''' + + if self.ton.using_validator(): + text += f"\nWallet: {self.wallet}" + + text += f''' Time: {time_} ({int(time.time())}) Alert name: {alert_name} Severity: {alert.severity} From dc7d068a02cf4a73ecebccf65b6530b30b23374b Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 26 Feb 2025 11:40:07 +0400 Subject: [PATCH 192/206] add logs if adnl connection failed in alerts --- modules/alert_bot.py | 1 + modules/utilities.py | 4 ++-- mytonctrl/mytonctrl.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 991118af..6802c027 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -362,6 +362,7 @@ def check_adnl_connection_failed(self): utils_module = UtilitiesModule(self.ton, self.local) ok, error = utils_module.check_adnl_connection() if not ok: + self.local.add_log(error, "warning") self.send_alert("adnl_connection_failed") def get_myself_from_election(self, config: dict): diff --git a/modules/utilities.py b/modules/utilities.py index 62c83efb..640f160d 100644 --- a/modules/utilities.py +++ b/modules/utilities.py @@ -355,7 +355,7 @@ def check_adnl_connection(self): response = requests.post(url, json=data, timeout=5).json() except Exception as e: ok = False - error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}' + error = f'Failed to check ADNL connection to local node: {type(e)}: {e}' continue result = response.get("ok") if result: @@ -363,7 +363,7 @@ def check_adnl_connection(self): break if not result: ok = False - error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}' + error = f'Failed to check ADNL connection to local node: {response.get("message")}' return ok, error def get_pool_data(self, args): diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index ffb7252b..1a2cfbae 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -488,6 +488,7 @@ def check_adnl(local, ton): utils_module = UtilitiesModule(ton, local) ok, error = utils_module.check_adnl_connection() if not ok: + error = "{red}" + error + "{endc}" print_warning(local, error) #end define From 638cf6c36c1e7ace3c2732db4cf61462d01bdbcf Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 26 Feb 2025 14:32:32 +0400 Subject: [PATCH 193/206] do not use masterchain_out_of_ser if state serializer is disabled --- modules/prometheus.py | 2 +- mytoncore/mytoncore.py | 1 + mytonctrl/mytonctrl.py | 8 ++++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/prometheus.py b/modules/prometheus.py index eac13f47..fe7d3d6c 100644 --- a/modules/prometheus.py +++ b/modules/prometheus.py @@ -44,7 +44,7 @@ def get_validator_status_metrics(self, result: list): result.append(METRICS['master_out_of_sync'].to_format(status.masterchain_out_of_sync)) if status.shardchain_out_of_sync is not None: result.append(METRICS['shard_out_of_sync'].to_format(status.shardchain_out_of_sync)) - if status.masterchain_out_of_ser is not None and status.stateserializermasterchainseqno != 0: + if status.stateserializerenabled and status.masterchain_out_of_ser is not None and status.stateserializermasterchainseqno != 0: result.append(METRICS['out_of_ser'].to_format(status.masterchain_out_of_ser)) if status.masterchainblock is not None and status.gcmasterchainblock is not None: result.append(METRICS['celldb_gc_block'].to_format(status.masterchainblock - status.gcmasterchainblock)) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index d78b46f2..64218b02 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -805,6 +805,7 @@ def GetValidatorStatus(self): status.out_of_sync = status.masterchain_out_of_sync if status.masterchain_out_of_sync > status.shardchain_out_of_sync else status.shardchain_out_of_sync status.out_of_ser = status.masterchain_out_of_ser status.last_deleted_mc_state = int(parse(result, "last_deleted_mc_state", '\n')) + status.stateserializerenabled = parse(result, "stateserializerenabled", '\n') == "true" self.local.try_function(self.parse_stats_from_vc, args=[result, status]) except Exception as ex: self.local.add_log(f"GetValidatorStatus warning: {ex}", "warning") diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 63f91c9e..c4479a79 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -746,7 +746,10 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, master_out_of_sync_text = local.translate("local_status_master_out_of_sync").format(GetColorInt(validator_status.masterchain_out_of_sync, 20, logic="less", ending=" sec")) shard_out_of_sync_text = local.translate("local_status_shard_out_of_sync").format(GetColorInt(validator_status.shardchain_out_of_sync, 5, logic="less", ending=" blocks")) - validator_out_of_ser_text = local.translate("local_status_validator_out_of_ser").format(f'{validator_status.out_of_ser} blocks ago') + validator_out_of_ser_text = None + + if validator_status.stateserializerenabled: + validator_out_of_ser_text = local.translate("local_status_validator_out_of_ser").format(f'{validator_status.out_of_ser} blocks ago') dbSize_text = GetColorInt(dbSize, 1000, logic="less", ending=" Gb") dbUsage_text = GetColorInt(dbUsage, 80, logic="less", ending="%") @@ -797,7 +800,8 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, print(validator_out_of_sync_text) print(master_out_of_sync_text) print(shard_out_of_sync_text) - print(validator_out_of_ser_text) + if validator_out_of_ser_text: + print(validator_out_of_ser_text) print(dbStatus_text) print(mtcVersion_text) print(validatorVersion_text) From af9cf220ac575190d29b96bfc20ef3bf7b9e5046 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 28 Feb 2025 12:04:41 +0400 Subject: [PATCH 194/206] add collated and validated blocks stats --- mytoncore/functions.py | 52 ++++++++++++++++++++++++++++++ mytonctrl/mytonctrl.py | 20 ++++++++++++ mytonctrl/resources/translate.json | 10 ++++++ 3 files changed, 82 insertions(+) diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 66080db6..1b65eb3b 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -286,6 +286,56 @@ def CalculateNetworkStatistics(zerodata, data): # end define +def save_node_statistics(local, ton): + status = ton.GetValidatorStatus() + if status.unixtime is None: + return + data = {'timestamp': status.unixtime} + + def get_ok_error(value: str): + ok, error = value.split() + return int(ok.split(':')[1]), int(error.split(':')[1]) + + if 'total.collated_blocks.master' in status: + master_ok, master_error = get_ok_error(status['total.collated_blocks.master']) + shard_ok, shard_error = get_ok_error(status['total.collated_blocks.shard']) + data['collated_blocks'] = { + 'master': {'ok': master_ok, 'error': master_error}, + 'shard': {'ok': shard_ok, 'error': shard_error}, + } + if 'total.validated_blocks.master' in status: + master_ok, master_error = get_ok_error(status['total.validated_blocks.master']) + shard_ok, shard_error = get_ok_error(status['total.validated_blocks.shard']) + data['validated_blocks'] = { + 'master': {'ok': master_ok, 'error': master_error}, + 'shard': {'ok': shard_ok, 'error': shard_error}, + } + if 'total.ext_msg_check' in status: + ok, error = get_ok_error(status['total.ext_msg_check']) + data['ext_msg_check'] = {'ok': ok, 'error': error} + if 'total.ls_queries_ok' in status and 'total.ls_queries_error' in status: + data['ls_queries'] = {} + for k in status['total.ls_queries_ok']: + if k.startswith('TOTAL'): + data['ls_queries']['ok'] = k.split(':')[1] + for k in status['total.ls_queries_error']: + if k.startswith('TOTAL'): + data['ls_queries']['error'] = k.split(':')[1] + statistics = local.db.get("statistics", dict()) + + # statistics['node'] = [stats_from_election_id, stats_from_prev_min, stats_now] + + election_id = ton.GetConfig32()['startWorkTime'] + if 'node' not in statistics or len(statistics['node']) < 1: + statistics['node'] = [data] + elif statistics['node'][0]['timestamp'] < election_id: + statistics['node'] = [data, data] + else: + statistics['node'] = statistics.get('node', []) + [data] + statistics['node'].pop(1) + local.db["statistics"] = statistics + + def ReadTransData(local, scanner): transData = local.buffer.transData SetToTimeData(transData, scanner.transNum) @@ -581,6 +631,8 @@ def General(local): local.start_cycle(ScanLiteServers, sec=60, args=(local, ton,)) + local.start_cycle(save_node_statistics, sec=60, args=(local, ton, )) + from modules.custom_overlays import CustomOverlayModule local.start_cycle(CustomOverlayModule(ton, local).custom_overlays, sec=60, args=()) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index c4479a79..63816fd3 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -751,6 +751,23 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, if validator_status.stateserializerenabled: validator_out_of_ser_text = local.translate("local_status_validator_out_of_ser").format(f'{validator_status.out_of_ser} blocks ago') + collated, validated = None, None + if ton.using_validator() and validatorIndex != -1: + stats = ton.local.db.get('statistics', {}).get('node') + if stats is not None and len(stats) == 3: + collated_ok = 0 + collated_error = 0 + validated_ok = 0 + validated_error = 0 + for k in ['master', 'shard']: + collated_ok += stats[2]['collated_blocks'][k]['ok'] - stats[0]['collated_blocks'][k]['ok'] + collated_error += stats[2]['collated_blocks'][k]['error'] - stats[0]['collated_blocks'][k]['error'] + validated_ok += stats[2]['validated_blocks'][k]['ok'] - stats[0]['validated_blocks'][k]['ok'] + validated_error += stats[2]['validated_blocks'][k]['error'] - stats[0]['validated_blocks'][k]['error'] + collated = local.translate['collated_blocks'].format(collated_ok, collated_error) + validated = local.translate['validated_blocks'].format(validated_ok, validated_error) + + dbSize_text = GetColorInt(dbSize, 1000, logic="less", ending=" Gb") dbUsage_text = GetColorInt(dbUsage, 80, logic="less", ending="%") dbStatus_text = local.translate("local_status_db").format(dbSize_text, dbUsage_text) @@ -800,6 +817,9 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, print(validator_out_of_sync_text) print(master_out_of_sync_text) print(shard_out_of_sync_text) + if collated and validated: + print(collated) + print(validated) if validator_out_of_ser_text: print(validator_out_of_ser_text) print(dbStatus_text) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index bb8bf284..e12736e2 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -344,6 +344,16 @@ "ru": "Серализация стейта локального валидатора была: {0}", "zh_TW": "本地驗證者最後一次狀態序列化: {0}" }, + "collated_blocks": { + "en": "Collated blocks since validation round started (success/error): {0}", + "ru": "Собранные блоки с начала раунда валидации (успех/ошибка): {0}", + "zh_TW": "自驗證週期開始以來的彙編區塊 (成功/錯誤): {0}" + }, + "validated_blocks": { + "en": "Validated blocks since validation round started (success/error): {0}", + "ru": "Проверенные блоки с начала раунда валидации (успех/ошибка): {0}", + "zh_TW": "自驗證週期開始以來的驗證區塊 (成功/錯誤): {0}" + }, "local_status_db": { "en": "Local validator database size: {0}, {1}", "ru": "Размер БД локального валидатора: {0}, {1}", From c779d2025099268e77a41d34f450622e306c8103 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 28 Feb 2025 14:00:36 +0400 Subject: [PATCH 195/206] refactor bot setup --- modules/__init__.py | 2 -- modules/alert_bot.py | 31 ++++++++++++++++-------------- mytonctrl/mytonctrl.py | 4 ---- mytonctrl/resources/translate.json | 5 +++++ 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/modules/__init__.py b/modules/__init__.py index ea5a85ef..cd1b0076 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -59,8 +59,6 @@ class Setting: 'defaultCustomOverlaysUrl': Setting(None, 'https://ton-blockchain.github.io/fallback_custom_overlays.json', 'Default custom overlays config url'), 'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'), 'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'), - 'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'), - 'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id'), 'auto_backup': Setting('validator', None, 'Make validator backup every election'), 'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'), 'prometheus_url': Setting('prometheus', None, 'Prometheus pushgateway url'), diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 6802c027..f39383c3 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -256,6 +256,22 @@ def test_alert(self, args): self.init() self.send_message('Test alert') + def setup_alert_bot(self, args): + if len(args) != 2: + raise Exception("Usage: setup_alert_bot ") + self.token = args[0] + self.chat_id = args[1] + init_alerts() + try: + self.send_welcome_message() + self.ton.local.db['BotToken'] = args[0] + self.ton.local.db['ChatId'] = args[1] + color_print("setup_alert_bot - {green}OK{endc}") + except Exception as e: + self.local.add_log(f"Error while sending welcome message: {e}", "error") + self.local.add_log(f"If you want the bot to write to a multi-person chat group, make sure the bot is added to that chat group. If it is not - do it and run the command `setup_alert_bot ` again.", "info") + color_print("setup_alert_bot - {red}Error{endc}") + def send_welcome_message(self): message = f""" This is alert bot. You have connected validator with ADNL {self.ton.GetAdnlAddr()}. @@ -275,20 +291,6 @@ def send_welcome_message(self): """ self.send_message(text=message, disable_web_page_preview=True) - def on_set_chat_id(self, chat_id): - self.token = self.ton.local.db.get("BotToken") - if self.token is None: - raise Exception("BotToken is not set") - self.chat_id = chat_id - init_alerts() - try: - self.send_welcome_message() - return True - except Exception as e: - self.local.add_log(f"Error while sending welcome message: {e}", "error") - self.local.add_log(f"If you want the bot to write to a multi-person chat group, make sure the bot is added to that chat group. If it is not - do it and run the command `set ChatId ` again.", "info") - return False - def check_db_usage(self): if time.time() - self.last_db_check < 600: return @@ -452,3 +454,4 @@ def add_console_commands(self, console): console.AddItem("disable_alert", self.disable_alert, self.local.translate("disable_alert_cmd")) console.AddItem("list_alerts", self.print_alerts, self.local.translate("list_alerts_cmd")) console.AddItem("test_alert", self.test_alert, self.local.translate("test_alert_cmd")) + console.AddItem("setup_alert_bot", self.setup_alert_bot, self.local.translate("setup_alert_bot_cmd")) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 283b7bba..8a7be630 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -896,10 +896,6 @@ def SetSettings(local, ton, args): color_print(f"{{red}} Error: set {name} ... is deprecated and does not work {{endc}}." f"\nInstead, use {{bold}}enable_mode {mode_name}{{endc}}") return - if name == 'ChatId': - from modules.alert_bot import AlertBotModule - if not AlertBotModule(ton, local).on_set_chat_id(value): - return force = False if len(args) > 2: if args[2] == "--force": diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index fbbbc1cf..0bdef05b 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -499,6 +499,11 @@ "ru": "Отправить тестовое оповещение через Telegram Bot", "zh_TW": "通過 Telegram Bot 發送測試警報" }, + "setup_alert_bot_cmd": { + "en": "Setup Telegram Bot for alerts", + "ru": "Настроить Telegram Bot для оповещений", + "zh_TW": "設置 Telegram Bot 以接收警報" + }, "benchmark_cmd": { "en": "Run benchmark", "ru": "Запустить бенчмарк", From f5cd1d3f13ef1c2e5059844c36445c2a306ab79a Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 28 Feb 2025 13:46:47 +0400 Subject: [PATCH 196/206] update node statistics collecting wip --- mytoncore/functions.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 1b65eb3b..0f60712f 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -315,22 +315,27 @@ def get_ok_error(value: str): data['ext_msg_check'] = {'ok': ok, 'error': error} if 'total.ls_queries_ok' in status and 'total.ls_queries_error' in status: data['ls_queries'] = {} - for k in status['total.ls_queries_ok']: + for k in status['total.ls_queries_ok'].split(): if k.startswith('TOTAL'): - data['ls_queries']['ok'] = k.split(':')[1] - for k in status['total.ls_queries_error']: + data['ls_queries']['ok'] = int(k.split(':')[1]) + for k in status['total.ls_queries_error'].split(): if k.startswith('TOTAL'): - data['ls_queries']['error'] = k.split(':')[1] + data['ls_queries']['error'] = int(k.split(':')[1]) statistics = local.db.get("statistics", dict()) # statistics['node'] = [stats_from_election_id, stats_from_prev_min, stats_now] - election_id = ton.GetConfig32()['startWorkTime'] - if 'node' not in statistics or len(statistics['node']) < 1: - statistics['node'] = [data] - elif statistics['node'][0]['timestamp'] < election_id: - statistics['node'] = [data, data] - else: + election_id = ton.GetConfig34()['startWorkTime'] + if 'node' not in statistics or len(statistics['node']) == 0: + statistics['node'] = [None, data] + elif len(statistics['node']) < 3: + statistics['node'].append(data) + if len(statistics['node']) == 3: + if statistics['node'][0] is None: + if 0 < data['timestamp'] - election_id < 90: + statistics['node'][0] = data + elif statistics['node'][0]['timestamp'] < election_id: + statistics['node'][0] = data statistics['node'] = statistics.get('node', []) + [data] statistics['node'].pop(1) local.db["statistics"] = statistics From 34c79704dc505aa9a75608e095f285958db9eef0 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 3 Mar 2025 11:56:37 +0400 Subject: [PATCH 197/206] print ls stats to status refactor --- mytoncore/functions.py | 2 +- mytoncore/mytoncore.py | 38 ++++++++++++++++++++++++++++-- mytonctrl/mytonctrl.py | 29 ++++++++++------------- mytonctrl/resources/translate.json | 17 ++++++++----- 4 files changed, 61 insertions(+), 25 deletions(-) diff --git a/mytoncore/functions.py b/mytoncore/functions.py index 0f60712f..df8b981c 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -287,7 +287,7 @@ def CalculateNetworkStatistics(zerodata, data): def save_node_statistics(local, ton): - status = ton.GetValidatorStatus() + status = ton.GetValidatorStatus(no_cache=True) if status.unixtime is None: return data = {'timestamp': status.unixtime} diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 64218b02..e63ed2bc 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -771,11 +771,11 @@ def parse_stats_from_vc(self, output: str, result: dict): if name not in result: result[name] = value - def GetValidatorStatus(self): + def GetValidatorStatus(self, no_cache=False): # Get buffer bname = "validator_status" buff = self.GetFunctionBuffer(bname) - if buff: + if buff and not no_cache: return buff #end if @@ -3038,6 +3038,40 @@ def GetStatistics(self, name, statistics=None): return data #end define + def get_node_statistics(self): + """ + :return: stats for collated/validated blocks since round beggining and stats for ls queries for the last minute + """ + stats = self.local.db.get('statistics', {}).get('node') + result = {} + if stats is not None and len(stats) == 3 and stats[0] is not None: + collated_ok = 0 + collated_error = 0 + validated_ok = 0 + validated_error = 0 + for k in ['master', 'shard']: + collated_ok += stats[2]['collated_blocks'][k]['ok'] - stats[0]['collated_blocks'][k]['ok'] + collated_error += stats[2]['collated_blocks'][k]['error'] - stats[0]['collated_blocks'][k]['error'] + validated_ok += stats[2]['validated_blocks'][k]['ok'] - stats[0]['validated_blocks'][k]['ok'] + validated_error += stats[2]['validated_blocks'][k]['error'] - stats[0]['validated_blocks'][k]['error'] + result = { + 'collated': { + 'ok': collated_ok, + 'error': collated_error, + }, + 'validated': { + 'ok': validated_ok, + 'error': validated_error, + }, + } + if stats is not None and len(stats) >= 2: + result['ls_queries'] = { + 'ok': stats[-1]['ls_queries']['ok'] - stats[-2]['ls_queries']['ok'], + 'error': stats[-1]['ls_queries']['error'] - stats[-2]['ls_queries']['error'], + 'time': stats[-1].get('timestamp', 0) - stats[-2].get('timestamp', 0), + } + return result + def GetSettings(self, name): # self.local.load_db() result = self.local.db.get(name) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 63816fd3..45597558 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -752,21 +752,16 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, validator_out_of_ser_text = local.translate("local_status_validator_out_of_ser").format(f'{validator_status.out_of_ser} blocks ago') collated, validated = None, None + ls_queries = None if ton.using_validator() and validatorIndex != -1: - stats = ton.local.db.get('statistics', {}).get('node') - if stats is not None and len(stats) == 3: - collated_ok = 0 - collated_error = 0 - validated_ok = 0 - validated_error = 0 - for k in ['master', 'shard']: - collated_ok += stats[2]['collated_blocks'][k]['ok'] - stats[0]['collated_blocks'][k]['ok'] - collated_error += stats[2]['collated_blocks'][k]['error'] - stats[0]['collated_blocks'][k]['error'] - validated_ok += stats[2]['validated_blocks'][k]['ok'] - stats[0]['validated_blocks'][k]['ok'] - validated_error += stats[2]['validated_blocks'][k]['error'] - stats[0]['validated_blocks'][k]['error'] - collated = local.translate['collated_blocks'].format(collated_ok, collated_error) - validated = local.translate['validated_blocks'].format(validated_ok, validated_error) - + node_stats = ton.get_node_statistics() + if node_stats and 'collated' in node_stats and 'validated' in node_stats: + collated = local.translate('collated_blocks').format(node_stats['collated']['ok'], node_stats['collated']['error']) + validated = local.translate('validated_blocks').format(node_stats['validated']['ok'], node_stats['validated']['error']) + if ton.using_liteserver(): + node_stats = ton.get_node_statistics() + if node_stats and 'ls_queries' in node_stats: + ls_queries = local.translate('ls_queries').format(node_stats['ls_queries']['time'], node_stats['ls_queries']['ok'], node_stats['ls_queries']['error']) dbSize_text = GetColorInt(dbSize, 1000, logic="less", ending=" Gb") dbUsage_text = GetColorInt(dbUsage, 80, logic="less", ending="%") @@ -817,11 +812,13 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, print(validator_out_of_sync_text) print(master_out_of_sync_text) print(shard_out_of_sync_text) + if validator_out_of_ser_text: + print(validator_out_of_ser_text) if collated and validated: print(collated) print(validated) - if validator_out_of_ser_text: - print(validator_out_of_ser_text) + if ls_queries: + print(ls_queries) print(dbStatus_text) print(mtcVersion_text) print(validatorVersion_text) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index e12736e2..241a0fae 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -345,14 +345,19 @@ "zh_TW": "本地驗證者最後一次狀態序列化: {0}" }, "collated_blocks": { - "en": "Collated blocks since validation round started (success/error): {0}", - "ru": "Собранные блоки с начала раунда валидации (успех/ошибка): {0}", - "zh_TW": "自驗證週期開始以來的彙編區塊 (成功/錯誤): {0}" + "en": "Collated blocks since validation round started (success/error): {0}/{1}", + "ru": "Собранные блоки с начала раунда валидации (успех/ошибка): {0}/{1}", + "zh_TW": "自驗證週期開始以來的彙編區塊 (成功/錯誤): {0}/{1}" }, "validated_blocks": { - "en": "Validated blocks since validation round started (success/error): {0}", - "ru": "Проверенные блоки с начала раунда валидации (успех/ошибка): {0}", - "zh_TW": "自驗證週期開始以來的驗證區塊 (成功/錯誤): {0}" + "en": "Validated blocks since validation round started (success/error): {0}/{1}", + "ru": "Проверенные блоки с начала раунда валидации (успех/ошибка): {0}/{1}", + "zh_TW": "自驗證週期開始以來的驗證區塊 (成功/錯誤): {0}/{1}" + }, + "ls_queries": { + "en": "Liteserver queries for the past {0} sec (success/error): {1}/{2}", + "ru": "Запросы к лайтсерверу за последние {0} сек (успех/ошибка): {1}/{2}", + "zh_TW": "過去 {0} 秒的輕量級服務器查詢 (成功/錯誤): {1}/{2}" }, "local_status_db": { "en": "Local validator database size: {0}, {1}", From 8180af539a791bb51f020a06ca4fbb152d9fb5b9 Mon Sep 17 00:00:00 2001 From: yungwine Date: Mon, 3 Mar 2025 13:25:17 +0400 Subject: [PATCH 198/206] add prometheus ls metrics --- modules/prometheus.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/modules/prometheus.py b/modules/prometheus.py index fe7d3d6c..6a48ac60 100644 --- a/modules/prometheus.py +++ b/modules/prometheus.py @@ -26,6 +26,12 @@ def to_format(self, value): 'stake': Metric('validator_stake', 'Validator stake', 'gauge'), 'celldb_gc_block': Metric('validator_celldb_gc_block', 'Celldb GC block latency', 'gauge'), 'celldb_gc_state': Metric('validator_celldb_gc_state', 'Celldb GC queue size', 'gauge'), + 'collated_master_ok': Metric('validator_blocks_collated_master_ok', 'Number of masterchain blocks successfully collated', 'gauge'), + 'collated_master_err': Metric('validator_blocks_collated_master_err', 'Number of masterchain blocks failed to collate', 'gauge'), + 'collated_shard_ok': Metric('validator_blocks_collated_shard_ok', 'Number of shardchain blocks successfully collated', 'gauge'), + 'collated_shard_err': Metric('validator_blocks_collated_shard_err', 'Number of shardchain blocks failed to collate', 'gauge'), + 'ls_queries_ok': Metric('validator_ls_queries_ok', 'Number of Liteserver successful queries', 'gauge'), + 'ls_queries_err': Metric('validator_ls_queries_err', 'Number of Liteserver failed queries', 'gauge'), } @@ -67,6 +73,15 @@ def get_validator_validation_metrics(self, result: list): if stake: result.append(METRICS['stake'].to_format(round(stake, 2))) + def get_node_stats_metrics(self, result: list): + stats = self.ton.get_node_statistics() + if stats and 'ls_queries' in stats: + if stats['ls_queries']['time'] < 50: + self.local.add_log(f'Liteserver queries time is too low: {stats}') + return + result.append(METRICS['ls_queries_ok'].to_format(stats['ls_queries']['ok'])) + result.append(METRICS['ls_queries_err'].to_format(stats['ls_queries']['error'])) + def push_metrics(self): if not self.ton.using_prometheus(): return @@ -77,6 +92,7 @@ def push_metrics(self): metrics = [] self.local.try_function(self.get_validator_status_metrics, args=[metrics]) self.local.try_function(self.get_validator_validation_metrics, args=[metrics]) + self.local.try_function(self.get_node_stats_metrics, args=[metrics]) requests.post(url, data='\n'.join(metrics).encode()) def add_console_commands(self, console): From 8f55bdc1b82779425ed110bbc684f4f25b4ffede Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 4 Mar 2025 13:06:08 +0400 Subject: [PATCH 199/206] send alert zero_block_created every 3 hours --- modules/alert_bot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index f39383c3..3b349029 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -76,9 +76,9 @@ def init_alerts(): ), "zero_block_created": Alert( "critical", - f"Validator has not created any blocks in the {int(VALIDATION_PERIOD // 3 // 3600)} hours", + f"Validator has not created any blocks in the {int(VALIDATION_PERIOD // 6 // 3600)} hours", "Validator has not created any blocks in the last {hours} hours.", - VALIDATION_PERIOD // 3 + VALIDATION_PERIOD // 6 ), "validator_slashed": Alert( "high", @@ -341,7 +341,7 @@ def check_zero_blocks_created(self): if not self.ton.using_validator(): return ts = get_timestamp() - period = VALIDATION_PERIOD // 3 # 6h for mainnet, 40m for testnet + period = VALIDATION_PERIOD // 6 # 3h for mainnet, 40m for testnet start, end = ts - period, ts - 60 config34 = self.ton.GetConfig34() if start < config34.startWorkTime: # round started recently From cb155a130930369c4399b0a6df79aa09be3db1ed Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 4 Mar 2025 13:17:06 +0400 Subject: [PATCH 200/206] upd numbers formatting --- modules/alert_bot.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 3b349029..18cde385 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -158,6 +158,11 @@ def send_alert(self, alert_name: str, *args, **kwargs): if alert is None: raise Exception(f"Alert {alert_name} not found") alert_name_readable = alert_name.replace('_', ' ').title() + + for key, value in kwargs.items(): + if isinstance(value, (int, float)): + kwargs[key] = f'{value:,}'.replace(',', ' ') # make space separator for thousands + text = '🆘' if alert.severity != 'info' else '' text += f''' Node {self.hostname}: {alert_name_readable} @@ -394,7 +399,7 @@ def check_stake_sent(self): if res is False: self.send_alert("stake_not_accepted") return - self.send_alert("stake_accepted", stake=round(res.get('stake'), 2)) + self.send_alert("stake_accepted", stake=round(res.get('stake'))) def check_stake_returned(self): if not self.ton.using_validator(): @@ -409,7 +414,7 @@ def check_stake_returned(self): for tr in trs: if tr.time >= config['endWorkTime'] + FREEZE_PERIOD and tr.srcAddr == '3333333333333333333333333333333333333333333333333333333333333333' and tr.body.startswith('F96F7324'): # Elector Recover Stake Response - self.send_alert("stake_returned", stake=round(tr.value, 2), address=res["walletAddr"], reward=round(tr.value - res.get('stake', 0), 2)) + self.send_alert("stake_returned", stake=round(tr.value), address=res["walletAddr"], reward=round(tr.value - res.get('stake', 0), 2)) return self.send_alert("stake_not_returned", address=res["walletAddr"]) From 53bf2f8fe48e37d956f2a907c4f9eda4d5c6620b Mon Sep 17 00:00:00 2001 From: yungwine Date: Tue, 4 Mar 2025 13:23:31 +0400 Subject: [PATCH 201/206] upd numbers formatting 2 --- modules/alert_bot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/alert_bot.py b/modules/alert_bot.py index 18cde385..1c88af8d 100644 --- a/modules/alert_bot.py +++ b/modules/alert_bot.py @@ -77,7 +77,7 @@ def init_alerts(): "zero_block_created": Alert( "critical", f"Validator has not created any blocks in the {int(VALIDATION_PERIOD // 6 // 3600)} hours", - "Validator has not created any blocks in the last {hours} hours.", + "Validator has not created any blocks in the last {hours} hours.", VALIDATION_PERIOD // 6 ), "validator_slashed": Alert( @@ -95,13 +95,13 @@ def init_alerts(): "stake_accepted": Alert( "info", "Validator's stake has been accepted (info alert with no sound)", - "Validator's stake {stake} TON has been accepted", + "Validator's stake {stake} TON has been accepted", ELECTIONS_START_BEFORE ), "stake_returned": Alert( "info", "Validator's stake has been returned (info alert with no sound)", - "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", + "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.", 60 ), "stake_not_returned": Alert( From 75a46adbe160d2e4955f982b1f001a2d3de1f3c2 Mon Sep 17 00:00:00 2001 From: yungwine Date: Wed, 5 Mar 2025 11:26:39 +0400 Subject: [PATCH 202/206] add validation prometheus metrics --- modules/prometheus.py | 19 +++++++++++++++++++ mytoncore/functions.py | 3 +++ mytoncore/mytoncore.py | 41 ++++++++++++++++++++++++++--------------- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/modules/prometheus.py b/modules/prometheus.py index 6a48ac60..e88e1737 100644 --- a/modules/prometheus.py +++ b/modules/prometheus.py @@ -30,6 +30,12 @@ def to_format(self, value): 'collated_master_err': Metric('validator_blocks_collated_master_err', 'Number of masterchain blocks failed to collate', 'gauge'), 'collated_shard_ok': Metric('validator_blocks_collated_shard_ok', 'Number of shardchain blocks successfully collated', 'gauge'), 'collated_shard_err': Metric('validator_blocks_collated_shard_err', 'Number of shardchain blocks failed to collate', 'gauge'), + 'validated_master_ok': Metric('validator_blocks_validated_master_ok', 'Number of masterchain blocks successfully validated', 'gauge'), + 'validated_master_err': Metric('validator_blocks_validated_master_err', 'Number of masterchain blocks failed to validate', 'gauge'), + 'validated_shard_ok': Metric('validator_blocks_validated_shard_ok', 'Number of shardchain blocks successfully validated', 'gauge'), + 'validated_shard_err': Metric('validator_blocks_validated_shard_err', 'Number of shardchain blocks failed to validate', 'gauge'), + 'validator_groups_master': Metric('validator_active_groups_master', 'Number of masterchain validation groups validator participates in', 'gauge'), + 'validator_groups_shard': Metric('validator_active_groups_shard', 'Number of shardchain validation groups validator participates in', 'gauge'), 'ls_queries_ok': Metric('validator_ls_queries_ok', 'Number of Liteserver successful queries', 'gauge'), 'ls_queries_err': Metric('validator_ls_queries_err', 'Number of Liteserver failed queries', 'gauge'), } @@ -59,6 +65,9 @@ def get_validator_status_metrics(self, result: list): result.append(METRICS['celldb_gc_state'].to_format(status.gcmasterchainblock - status.last_deleted_mc_state)) else: result.append(METRICS['celldb_gc_state'].to_format(-1)) + if status.validator_groups_master is not None: + result.append(METRICS['validator_groups_master'].to_format(status.validator_groups_master)) + result.append(METRICS['validator_groups_shard'].to_format(status.validator_groups_shard)) result.append(METRICS['vc_up'].to_format(int(is_working))) def get_validator_validation_metrics(self, result: list): @@ -81,6 +90,16 @@ def get_node_stats_metrics(self, result: list): return result.append(METRICS['ls_queries_ok'].to_format(stats['ls_queries']['ok'])) result.append(METRICS['ls_queries_err'].to_format(stats['ls_queries']['error'])) + if stats and 'collated' in stats: + result.append(METRICS['collated_master_ok'].to_format(stats['collated']['master']['ok'])) + result.append(METRICS['collated_master_err'].to_format(stats['collated']['master']['error'])) + result.append(METRICS['collated_shard_ok'].to_format(stats['collated']['shard']['ok'])) + result.append(METRICS['collated_shard_err'].to_format(stats['collated']['shard']['error'])) + if stats and 'validated' in stats: + result.append(METRICS['validated_master_ok'].to_format(stats['validated']['master']['ok'])) + result.append(METRICS['validated_master_err'].to_format(stats['validated']['master']['error'])) + result.append(METRICS['validated_shard_ok'].to_format(stats['validated']['shard']['ok'])) + result.append(METRICS['validated_shard_err'].to_format(stats['validated']['shard']['error'])) def push_metrics(self): if not self.ton.using_prometheus(): diff --git a/mytoncore/functions.py b/mytoncore/functions.py index df8b981c..c39f2c42 100755 --- a/mytoncore/functions.py +++ b/mytoncore/functions.py @@ -323,6 +323,9 @@ def get_ok_error(value: str): data['ls_queries']['error'] = int(k.split(':')[1]) statistics = local.db.get("statistics", dict()) + if time.time() - int(status.start_time) <= 60: # was node restart <60 sec ago, resetting node statistics + statistics['node'] = [] + # statistics['node'] = [stats_from_election_id, stats_from_prev_min, stats_now] election_id = ton.GetConfig34()['startWorkTime'] diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index e63ed2bc..5ebee930 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -807,6 +807,10 @@ def GetValidatorStatus(self, no_cache=False): status.last_deleted_mc_state = int(parse(result, "last_deleted_mc_state", '\n')) status.stateserializerenabled = parse(result, "stateserializerenabled", '\n') == "true" self.local.try_function(self.parse_stats_from_vc, args=[result, status]) + if 'active_validator_groups' in status: + groups = status.active_validator_groups.split() # master:1 shard:2 + status.validator_groups_master = int(groups[0].split(':')[1]) + status.validator_groups_shard = int(groups[1].split(':')[1]) except Exception as ex: self.local.add_log(f"GetValidatorStatus warning: {ex}", "warning") status.is_working = False @@ -3043,27 +3047,34 @@ def get_node_statistics(self): :return: stats for collated/validated blocks since round beggining and stats for ls queries for the last minute """ stats = self.local.db.get('statistics', {}).get('node') - result = {} + result = { + 'collated': { + 'ok': 0, + 'error': 0, + }, + 'validated': { + 'ok': 0, + 'error': 0, + } + } if stats is not None and len(stats) == 3 and stats[0] is not None: - collated_ok = 0 - collated_error = 0 - validated_ok = 0 - validated_error = 0 for k in ['master', 'shard']: - collated_ok += stats[2]['collated_blocks'][k]['ok'] - stats[0]['collated_blocks'][k]['ok'] - collated_error += stats[2]['collated_blocks'][k]['error'] - stats[0]['collated_blocks'][k]['error'] - validated_ok += stats[2]['validated_blocks'][k]['ok'] - stats[0]['validated_blocks'][k]['ok'] - validated_error += stats[2]['validated_blocks'][k]['error'] - stats[0]['validated_blocks'][k]['error'] - result = { - 'collated': { + collated_ok = stats[2]['collated_blocks'][k]['ok'] - stats[0]['collated_blocks'][k]['ok'] + collated_error = stats[2]['collated_blocks'][k]['error'] - stats[0]['collated_blocks'][k]['error'] + validated_ok = stats[2]['validated_blocks'][k]['ok'] - stats[0]['validated_blocks'][k]['ok'] + validated_error = stats[2]['validated_blocks'][k]['error'] - stats[0]['validated_blocks'][k]['error'] + result['collated'][k] = { 'ok': collated_ok, 'error': collated_error, - }, - 'validated': { + } + result['validated'][k] = { 'ok': validated_ok, 'error': validated_error, - }, - } + } + result['collated']['ok'] += collated_ok + result['collated']['error'] += collated_error + result['validated']['ok'] += validated_ok + result['validated']['error'] += validated_error if stats is not None and len(stats) >= 2: result['ls_queries'] = { 'ok': stats[-1]['ls_queries']['ok'] - stats[-2]['ls_queries']['ok'], From 66b98b8d1b900aba3eaab4121d87674999e199ab Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 7 Mar 2025 11:35:25 +0400 Subject: [PATCH 203/206] fix get_node_statistics --- mytoncore/mytoncore.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 5ebee930..54af3f22 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3047,18 +3047,19 @@ def get_node_statistics(self): :return: stats for collated/validated blocks since round beggining and stats for ls queries for the last minute """ stats = self.local.db.get('statistics', {}).get('node') - result = { - 'collated': { - 'ok': 0, - 'error': 0, - }, - 'validated': { - 'ok': 0, - 'error': 0, - } - } + result = {} if stats is not None and len(stats) == 3 and stats[0] is not None: for k in ['master', 'shard']: + result = { + 'collated': { + 'ok': 0, + 'error': 0, + }, + 'validated': { + 'ok': 0, + 'error': 0, + } + } collated_ok = stats[2]['collated_blocks'][k]['ok'] - stats[0]['collated_blocks'][k]['ok'] collated_error = stats[2]['collated_blocks'][k]['error'] - stats[0]['collated_blocks'][k]['error'] validated_ok = stats[2]['validated_blocks'][k]['ok'] - stats[0]['validated_blocks'][k]['ok'] @@ -3075,7 +3076,7 @@ def get_node_statistics(self): result['collated']['error'] += collated_error result['validated']['ok'] += validated_ok result['validated']['error'] += validated_error - if stats is not None and len(stats) >= 2: + if stats is not None and len(stats) >= 2 and stats[0] is not None: result['ls_queries'] = { 'ok': stats[-1]['ls_queries']['ok'] - stats[-2]['ls_queries']['ok'], 'error': stats[-1]['ls_queries']['error'] - stats[-2]['ls_queries']['error'], From 099ebb4dc5fc47567a8ea9712c8df499e84b62d5 Mon Sep 17 00:00:00 2001 From: yungwine Date: Fri, 7 Mar 2025 11:49:41 +0400 Subject: [PATCH 204/206] add active_validator_groups to status --- mytonctrl/mytonctrl.py | 7 +++++++ mytonctrl/resources/translate.json | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index 45597558..ac994ce8 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -751,6 +751,11 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, if validator_status.stateserializerenabled: validator_out_of_ser_text = local.translate("local_status_validator_out_of_ser").format(f'{validator_status.out_of_ser} blocks ago') + active_validator_groups = None + + if ton.using_validator() and validator_status.validator_groups_master and validator_status.validator_groups_shard: + active_validator_groups = local.translate("active_validator_groups").format(validator_status.validator_groups_master, validator_status.validator_groups_shard) + collated, validated = None, None ls_queries = None if ton.using_validator() and validatorIndex != -1: @@ -814,6 +819,8 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, print(shard_out_of_sync_text) if validator_out_of_ser_text: print(validator_out_of_ser_text) + if active_validator_groups: + print(active_validator_groups) if collated and validated: print(collated) print(validated) diff --git a/mytonctrl/resources/translate.json b/mytonctrl/resources/translate.json index 241a0fae..d92e5070 100644 --- a/mytonctrl/resources/translate.json +++ b/mytonctrl/resources/translate.json @@ -344,6 +344,11 @@ "ru": "Серализация стейта локального валидатора была: {0}", "zh_TW": "本地驗證者最後一次狀態序列化: {0}" }, + "active_validator_groups": { + "en": "Active validator groups (masterchain,shardchain): {0},{1}", + "ru": "Активные группы валидатора (мастерчейн, шардчейн): {0},{1}", + "zh_TW": "活动验证器组（主链、碎片链: {0},{1}" + }, "collated_blocks": { "en": "Collated blocks since validation round started (success/error): {0}/{1}", "ru": "Собранные блоки с начала раунда валидации (успех/ошибка): {0}/{1}", From a64237452f90b17826e5991f4b498bf6f437cd87 Mon Sep 17 00:00:00 2001 From: yungwine Date: Sat, 8 Mar 2025 21:30:28 +0100 Subject: [PATCH 205/206] update collated_blocks in status --- mytonctrl/mytonctrl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mytonctrl/mytonctrl.py b/mytonctrl/mytonctrl.py index ac994ce8..19f6ec43 100755 --- a/mytonctrl/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -758,11 +758,14 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency, collated, validated = None, None ls_queries = None - if ton.using_validator() and validatorIndex != -1: + if ton.using_validator(): node_stats = ton.get_node_statistics() if node_stats and 'collated' in node_stats and 'validated' in node_stats: collated = local.translate('collated_blocks').format(node_stats['collated']['ok'], node_stats['collated']['error']) validated = local.translate('validated_blocks').format(node_stats['validated']['ok'], node_stats['validated']['error']) + else: + collated = local.translate('collated_blocks').format('collecting data...', 'wait for the next validation round') + validated = local.translate('validated_blocks').format('collecting data...', 'wait for the next validation round') if ton.using_liteserver(): node_stats = ton.get_node_statistics() if node_stats and 'ls_queries' in node_stats: From d0dacc977a6304533e532922b676bb1f7227ccad Mon Sep 17 00:00:00 2001 From: Philip Wang Date: Fri, 4 Apr 2025 15:53:44 +0800 Subject: [PATCH 206/206] fix required_balance_for_loan credit decimal --- mytoncore/mytoncore.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mytoncore/mytoncore.py b/mytoncore/mytoncore.py index 00b1a7b2..31190663 100644 --- a/mytoncore/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -3625,7 +3625,8 @@ def GetController(self, mode): #end define def GetControllerRequiredBalanceForLoan(self, controllerAddr, credit, interest): - cmd = f"runmethodfull {controllerAddr} required_balance_for_loan {credit} {interest}" + credit_nano_tons = credit * 10**9 + cmd = f"runmethodfull {controllerAddr} required_balance_for_loan {credit_nano_tons} {interest}" result = self.liteClient.Run(cmd) data = self.Result2List(result) if data is None: