nesa/bootstrap.sh
2024-10-17 16:46:13 +00:00

1152 lines
38 KiB
Bash
Raw Permalink Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/bash
#
#
# ▄▄▄▄▄▄
# ███▀▀▀██▄ nesaorg/bootstrap
# ███ ███ ███████ ███████ █████
# ███ ███ ██ ██ ██ ██
# ▄▄▄ ███ █████ ███████ ███████
# ███ ███ ██ ██ ██ ██
# ███ ███ ███████ ███████ ██ ██
# bootstrap.sh fielding@nesa.ai
#
#
# noteworthy conventions: variables that are exported to the config file or the container environment files are in all caps
#
# vars
#
trap 'trap " " SIGINT SIGTERM SIGHUP; kill 0; wait; sigterm_handler' SIGINT SIGTERM SIGHUP
sigterm_handler() {
printf "\n Aborting node setup. Cleaning up...\n"
# Add any additional cleanup tasks here
echo; exit 1
}
# set -x
terminal_size=$(stty size)
terminal_height=${terminal_size% *}
terminal_width=${terminal_size#* }
prompt_height=${PROMPT_HEIGHT:-1}
main_color=43
link_color=69
logo=$(gum style ' ▄▄▄▄▄▄
███▀▀▀██▄
███ ███
███ ███
▄▄▄ ███
███ ███
███ ███')
CHAIN_ID="nesa-testnet-3"
domain="test.nesa.sh"
chain_container="ghcr.io/nesaorg/nesachain:testnet-latest"
import_key_expect_url="https://raw.githubusercontent.com/nesaorg/bootstrap/master/import_key.expect"
node_id_file="$HOME/.nesa/identity/node_id.id"
miner_type_none=0
miner_type_non_distributed=1
miner_type_distributed=2
miner_type_agnostic=3
distributed_type_none=0
distributed_type_new_swarm=1
distributed_type_existing_swarm=2
distributed_type_agnostic=3
# this will never load from the env file, but if they know to override it via ENV vars then they can
WORKING_DIRECTORY=${WORKING_DIRECTORY:-"$HOME/.nesa"}
env_dir="$WORKING_DIRECTORY/env"
agent_env_file="$env_dir/agent.env"
bsns_s_env_file="$env_dir/bsns-s.env"
bsns_c_env_file="$env_dir/bsns-c.env"
orchestrator_env_file="$env_dir/orchestrator.env"
fluentbit_env_file="$env_dir/fluentbit.env"
base_env_file="$env_dir/base.env"
config_env_file="$env_dir/.env"
init_pwd=$PWD # so they can get back to where they started!
status="booting" # lol not really doing anything with this currently
ORC_PORT=31333
MONIKER=${MONIKER:-$(hostname -s)}
#
# basic helper functions
#
# print if the output if ts on screen
print_test() {
local no_color
local max_length
max_length=$(max_line_length "$no_color")
no_color=$(printf '%b' "${1}" | sed -e 's/\x1B\[[0-9;]*[JKmsu]//g')
[ "$(printf '%s' "${no_color}" | wc -l)" -gt $((terminal_height - prompt_height)) ] && return 1
[ "$max_length" -gt "$terminal_width" ] && return 1
gum style --align center --width="${terminal_width}" "${1}" ''
printf '%b' "\033[A"
}
update_header() {
local dashboard_url
local op_dashboard_url
local public_key
if [[ "$NODE_ID" == "pending..." ]]; then
dashboard_url="https://node.nesa.ai"
else
dashboard_url="https://node.nesa.ai/nodes/$NODE_ID"
fi
if [[ -n "$NODE_PRIV_KEY" ]]; then
public_key=$(generate_public_key "$NODE_PRIV_KEY")
op_dashboard_url="https://node.nesa.ai/$public_key/list"
else
public_key="pending..."
op_dashboard_url="pending..."
fi
info=$(gum style "[1;38;5;${main_color}m ${MONIKER}.${domain}
----------------
[1;38;5;${main_color}mnode id: ${NODE_ID}
[1;38;5;${main_color}mpublic key: ${public_key}
[1;38;5;${main_color}mdashboard: [0;38;5;${link_color}m${dashboard_url}
[1;38;5;${main_color}mop dash: [0;38;5;${link_color}m${op_dashboard_url}
[1;38;5;${main_color}mstatus: ${status}")
header=$(gum join --horizontal --align top "${logo}" ' ' "${info}")
echo -e "\n"
print_test "${header}"
echo -e "\n"
}
# check if a command exists
command_exists() {
command -v "$1" >/dev/null 2>&1
}
# error handling function
handle_install_failure() {
echo "Failed to install Gum using available methods due to permissions or unsupported OS..."
echo "Please install Gum manually by visiting: https://github.com/charmbracelet/gum"
exit 1
}
# install gum using Go
install_gum_go() {
echo "Installing Gum using Go..."
go install github.com/charmbracelet/gum@latest || handle_install_failure
}
# install gum based on the operating system and availability of Go
install_gum() {
# Try to install using Go if available
if command_exists go; then
install_gum_go
return
fi
case "$(uname -s)" in
Darwin)
echo "Installing Gum using Homebrew..."
brew install gum || handle_install_failure
;;
Linux)
if command_exists apt-get; then
echo "Installing Gum on Ubuntu/Debian..."
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://repo.charm.sh/apt/gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/charm.gpg
echo "deb [signed-by=/etc/apt/keyrings/charm.gpg] https://repo.charm.sh/apt/ * *" | sudo tee /etc/apt/sources.list.d/charm.list
sudo apt update && sudo apt install gum || handle_install_failure
elif command_exists pacman; then
echo "Installing Gum using pacman..."
sudo pacman -S gum || handle_install_failure
elif command_exists nix-env; then
echo "Installing Gum using Nix..."
nix-env -iA nixpkgs.gum || handle_install_failure
else
handle_install_failure
fi
;;
CYGWIN* | MINGW32* | MSYS* | MINGW*)
if command_exists winget; then
echo "Installing Gum using WinGet..."
winget install charmbracelet.gum || handle_install_failure
elif command_exists scoop; then
echo "Installing Gum using Scoop..."
scoop install charm-gum || handle_install_failure
else
handle_install_failure
fi
;;
*)
handle_install_failure
;;
esac
}
check_gum_installed() {
if ! command_exists gum; then
echo "Attempting to install gum..."
install_gum
fi
}
check_jq_installed() {
if ! command -v jq &>/dev/null; then
install_jq
fi
}
install_jq() {
case "$(uname -s)" in
Linux)
if command -v apt-get &>/dev/null; then
gum spin -s line --title "Installing jq with apt-get..." -- sudo apt-get update && sudo apt-get install -y jq
elif command -v yum &>/dev/null; then
gum spin -s line --title "Installing jq with yum..." -- sudo yum install -y jq
elif command -v pacman &>/dev/null; then
gum spin -s line --title "Installing jq with pacman..." -- sudo pacman -Sy jq
elif command -v zypper &>/dev/null; then
gum spin -s line --title "Installing jq with zypper..." -- sudo zypper install -y jq
elif command -v dnf &>/dev/null; then
gum spin -s line --title "Installing jq with dnf..." -- sudo dnf install -y jq
else
echo "Package manager not found. Please install jq manually."
exit 1
fi
;;
Darwin)
if command -v brew &>/dev/null; then
gum spin -s line --title "Installing jq with brew..." -- brew install jq
else
echo "Homebrew is not installed. Please install jq manually."
exit 1
fi
;;
*)
echo "Unsupported OS. Please install jq manually."
exit 1
;;
esac
}
# check if Docker is installed
check_docker_installed() {
if ! command_exists docker; then
echo "Docker is not installed. Please install Docker and try again."
exit 1
fi
}
check_python_and_ecdsa() {
if ! command -v python3 &>/dev/null; then
echo "Python 3 is not installed. Please install Python 3 and try again."
exit 1
fi
if ! python3 -c "import ecdsa" &>/dev/null; then
echo "The ecdsa library is not installed. Installing now..."
pip3 install ecdsa
fi
}
# TODO: handle the need for sudo here -.-
# check_nvidia_installed() {
# if ! command_exists nvidia-smi; then
# echo "NVIDIA drivers are not installed. Please install NVIDIA drivers and try again."
# exit 1
# fi
# if ! command_exists nvidia-container-runtime; then
# sudo curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
# && sudo curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
# sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
# sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
# sudo apt-get update
# sudo apt-get install -y nvidia-container-toolkit
# sudo nvidia-ctk runtime configure --runtime=docker
# sudo systemctl restart docker
# fi
# }
# calculate max line length of the input
max_line_length() {
local max_len
local line_len
max_len=0
IFS=$'\n'
for line in $1; do
line_len=${#line}
if ((line_len > max_len)); then
max_len=$line_len
fi
done
echo "$max_len"
}
download_import_key_expect() {
# curl -o import_key.expect "$IMPORT_KEY_EXPECT_URL"
cp import_key.expect "$WORKING_DIRECTORY/"
chmod +x "$WORKING_DIRECTORY/import_key.expect"
}
get_linux_info() {
local name version kernel architecture cpu cores ram disk_avail gpu gpu_count gpu_memory
if [ -f /etc/os-release ]; then
. /etc/os-release
name=$NAME
version=$VERSION
else
name="Not Available"
version="Not Available"
fi
kernel=$(uname -r)
architecture=$(uname -m)
cpu=$(lscpu | grep 'Model name' | awk -F: '{print $2}' | sed 's/^ *//')
cores=$(lscpu | grep '^CPU(s):' | awk '{print $2}')
ram=$(free -h | grep Mem | awk '{print $2}')
disk_avail=$(df -h --total | grep total | awk '{print $4}')
gpu=$(lspci | grep -i -e '3D controller' -e 'VGA compatible controller' | grep -i -e nvidia -e amd | awk -F: '{print $3}' | sed 's/^ *//')
gpu_count=$(lspci | grep -i -e '3D controller' -e 'VGA compatible controller' | grep -i -e nvidia -e amd | wc -l | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
gpu_memory=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | awk '{total += $1} END {print total " MB"}')
if [ -z "$gpu_memory" ]; then
gpu_memory=$(lshw -C display 2>/dev/null | grep -i size | awk '{print $2 " " $3}' | head -n 1)
fi
NODE_OS="Linux $version"
NODE_ARCH="$architecture"
NODE_CPU="$cpu"
NODE_CORES="$cores"
NODE_GPU="${gpu:-NA}"
NODE_GPU_COUNT="${gpu_count:-0}"
NODE_RAM="$ram"
NODE_VRAM="${gpu_memory:-NA}"
NODE_DISK_AVAIL="$disk_avail"
}
get_macos_info() {
local product_version build_version architecture cpu cores ram disk_avail gpu gpu_count gpu_memory
product_version=$(sw_vers -productVersion)
build_version=$(sw_vers -buildVersion)
architecture=$(uname -m)
cpu=$(sysctl -n machdep.cpu.brand_string)
cores=$(sysctl -n hw.ncpu)
ram=$(sysctl -n hw.memsize | awk '{print $1/1024/1024/1024 " GB"}')
disk_avail=$(df -h / | grep / | awk '{print $4}')
gpu=$(system_profiler SPDisplaysDataType | grep 'Chipset Model' | awk -F: '{print $2}' | sed 's/^ *//')
gpu_count=$(system_profiler SPDisplaysDataType | grep 'Chipset Model' | wc -l | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
gpu_memory=$(system_profiler SPDisplaysDataType | grep 'VRAM' | awk -F: '{total += $2} END {print total " MB"}' | sed 's/^ *//')
NODE_OS="MacOS $product_version ($build_version)"
NODE_ARCH="$architecture"
NODE_CPU="$cpu"
NODE_CORES="$cores"
NODE_GPU="${gpu:-NA}"
NODE_GPU_COUNT="${gpu_count:-0}"
NODE_RAM="$ram"
NODE_VRAM="${gpu_memory:-NA}"
NODE_DISK_AVAIL="$disk_avail"
}
get_windows_info() {
local caption version architecture cpu cores ram disk_avail gpu gpu_count gpu_memory
caption=$(wmic os get Caption /value | awk -F= '{print $2}')
version=$(wmic os get Version /value | awk -F= '{print $2}')
architecture=$(wmic os get OSArchitecture /value | awk -F= '{print $2}')
cpu=$(wmic cpu get name /value | awk -F= '{print $2}')
cores=$(wmic cpu get NumberOfCores /value | awk -F= '{print $2}')
ram=$(wmic computersystem get totalphysicalmemory /value | awk -F= '{print $2/1024/1024/1024 " GB"}')
disk_avail=$(wmic logicaldisk get size,freespace,caption | awk '{if ($1 == "C:") print $3/1024/1024/1024 " GB"}')
gpu=$(wmic path win32_videocontroller get name /value | awk -F= '{print $2}')
gpu_count=$(wmic path win32_videocontroller get name /value | grep -c "Name" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
gpu_memory=$(wmic path win32_videocontroller get AdapterRAM /value | awk -F= '{total += $2} END {print total/1024/1024 " MB"}')
NODE_OS="Windows $caption $version"
NODE_ARCH="$architecture"
NODE_CPU="$cpu"
NODE_CORES="$cores"
NODE_GPU="${gpu:-NA}"
NODE_GPU_COUNT="${gpu_count:-0}"
NODE_RAM="$ram"
NODE_VRAM="${gpu_memory:-NA}"
NODE_DISK_AVAIL="$disk_avail"
}
get_wsl_info() {
local name version kernel architecture cpu cores ram disk_avail gpu gpu_count gpu_memory
if [ -f /etc/os-release ]; then
. /etc/os-release
name=$NAME
version=$VERSION
else
name="Not Available"
version="Not Available"
fi
kernel=$(uname -r)
architecture=$(uname -m)
cpu=$(grep -m1 'model name' /proc/cpuinfo | awk -F: '{print $2}' | sed 's/^ *//')
cores=$(grep -c '^processor' /proc/cpuinfo)
ram=$(free -h | grep Mem | awk '{print $2}')
disk_avail=$(df -h --total | grep total | awk '{print $4}')
if command -v nvidia-smi &> /dev/null; then
gpu=$(nvidia-smi --query-gpu=name --format=csv,noheader)
gpu_count=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
gpu_memory=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | awk '{total += $1} END {print total " MB"}')
else
gpu="NA"
gpu_count=0
gpu_memory="NA"
fi
NODE_OS="WSL $version ($kernel)"
NODE_ARCH="$architecture"
NODE_CPU="$cpu"
NODE_CORES="$cores"
NODE_GPU="${gpu:-NA}"
NODE_GPU_COUNT="${gpu_count:-0}"
NODE_RAM="$ram"
NODE_VRAM="${gpu_memory:-NA}"
NODE_DISK_AVAIL="$disk_avail"
}
detect_hardware_capabilities() {
case "$(uname -s)" in
Linux)
if grep -q Microsoft /proc/version; then
get_wsl_info
else
get_linux_info
fi
;;
Darwin)
get_macos_info
;;
CYGWIN*|MINGW*|MSYS*)
get_windows_info
;;
*)
echo "Unsupported platform"
;;
esac
}
setup_work_dir() {
if [ ! -d "$WORKING_DIRECTORY" ]; then
mkdir -p "$WORKING_DIRECTORY"
fi
cd "$WORKING_DIRECTORY" || {
echo -e "Error changing to working directory: $WORKING_DIRECTORY"
exit 1
}
setup_docker_repository
}
setup_docker_repository() {
# Clone or pull the latest changes if the repo already exists
if [ ! -d "docker" ]; then
gum spin -s line --title "Cloning the nesaorg/docker repository..." -- git clone https://github.com/nesaorg/docker.git
else
cd docker
gum spin -s line --title "Pulling latest updates from nesaorg/docker repository..." -- git pull
cd ..
fi
# Create symlink for env directory
if [ -d "docker" ]; then
ln -sfn "$env_dir" "docker/env"
else
echo "Error: Docker directory does not exist."
exit 1
fi
}
get_swarms_map() {
local url="https://lcd.test.nesa.ai/nesachain/dht/get_orchestrators"
local json_data
local excluded_node_ids
local exclude_node_ids_json
local map=()
excluded_node_ids=(
"QmbtSFavybyKNkP2MAhVftA4S7tAW5HXvbKGiX9hHx9XqF|mistralai|Mixtral-8x7B-Instruct-v0.1"
"QmR58ndfebR3LXNxT5qx3FgXMkb4AptjpDM83r1CXfAhAw|mistralai|Mixtral-8x7B-Instruct-v0.1"
"Qmc6GZVS41EzzU5j13cy1pL3HjhwJfaf1N71cjp2zt18HX|Orenguteng|Llama-3-8B-Lexi-Uncensored"
"QmR4Gi37D1cPnihhkvRG9kRGtBtXYxwAYo92x6y1FYxmij|bigscience|bloom-560m"
"QmeCvBP1N3BqDiQc7hGxNFgrtguVHncqGKChJJeMZtsM8C|randommodel"
"QmUxwnuEKAEY9CnB4tEPKvmwK6h6pmuSN3V28vQ9A3s8qQ|randommodel22"
)
exclude_node_ids_json=$(printf '%s\n' "${excluded_node_ids[@]}" | jq -R . | jq -s .)
json_data=$(curl -s "$url")
map=$(echo "$json_data" | jq -r --argjson exclude_node_ids "$exclude_node_ids_json" '
.orchestrators |
map(select(.node_id | (contains("/") | not))) |
map(select(.node_id as $id | $exclude_node_ids | index($id) | not)) |
map(
{
"node_id": (.node_id | split("|")[0]),
"organization": (.node_id | split("|")[1]),
"model_name": (.node_id | split("|")[2]),
"model_id": ((.node_id | split("|")[1]) + "/" + (.node_id | split("|")[2]))
}
)
')
echo "$map"
}
get_model_names() {
local map="$1"
local model_names
model_names=$(echo "$map" | jq -r '.[] | .model_id' | sort | uniq)
echo "$model_names"
}
get_node_id() {
local map="$1"
local model_id="$2"
local node_id
node_id=$(echo "$map" | jq -r --arg model_id "$model_id" '
.[] | select(.model_id == $model_id) | .node_id
')
echo "$node_id"
}
create_combined_node_id() {
local map="$1"
local model_id="$2"
local node_info
node_info=$(echo "$map" | jq -r --arg model_id "$model_id" '
.[] | select(.model_id == $model_id) | "\(.node_id)|\(.organization)|\(.model_name)"
')
echo "$node_info"
}
fetch_network_address() {
local recreated_node_id="$1"
local url="https://lcd.test.nesa.ai/nesachain/dht/get_node/$recreated_node_id"
local json_data
local network_address
json_data=$(curl -s "$url")
network_address=$(echo "$json_data" | jq -r '.node.network_address')
echo "$network_address"
}
generate_public_key() {
local private_key="$1"
python3 -c "
import ecdsa
def strip_0x_prefix(key_hex):
return key_hex[2:] if key_hex.startswith('0x') else key_hex
def private_key_to_public_key(private_key_hex):
private_key_hex = strip_0x_prefix(private_key_hex)
private_key_bytes = bytes.fromhex(private_key_hex)
sk = ecdsa.SigningKey.from_string(private_key_bytes, curve=ecdsa.SECP256k1)
vk = sk.get_verifying_key()
public_key_compressed = b'\x02' + vk.to_string()[:32] if vk.to_string()[-1] % 2 == 0 else b'\x03' + vk.to_string()[:32]
return public_key_compressed.hex()
print(private_key_to_public_key('$private_key'))
"
}
update_config_var() {
local file=$1
local var=$2
local value=$3
local temp_file
temp_file=$(mktemp)
if grep -q "^$var=" "$file"; then
# Use a temporary file to handle sed differences
sed "s|^$var=.*|$var=\"$value\"|" "$file" > "$temp_file" && mv "$temp_file" "$file"
else
echo "$var=\"$value\"" >> "$file"
fi
# Clean up the temporary file if it still exists
[ -f "$temp_file" ] && rm "$temp_file"
}
strip_0x_prefix() {
local key="$1"
# Remove 0x prefix if present
echo "${key#0x}"
}
save_to_env_file() {
# Config environment variables
update_config_var "$config_env_file" "IS_CHAIN" "$IS_CHAIN"
update_config_var "$config_env_file" "IS_VALIDATOR" "$IS_VALIDATOR"
update_config_var "$config_env_file" "IS_MINER" "$IS_MINER"
update_config_var "$config_env_file" "MINER_TYPE" "$MINER_TYPE"
update_config_var "$config_env_file" "DISTRIBUTED_TYPE" "$DISTRIBUTED_TYPE"
# Agent environment variables
update_config_var "$agent_env_file" "VIRTUAL_HOST" "$NODE_HOSTNAME"
update_config_var "$agent_env_file" "LETSENCRYPT_HOST" "$NODE_HOSTNAME"
update_config_var "$agent_env_file" "LETSENCRYPT_EMAIL" "$OP_EMAIL"
update_config_var "$agent_env_file" "CHAIN_ID" "$CHAIN_ID"
update_config_var "$agent_env_file" "NODE_HOSTNAME" "$NODE_HOSTNAME"
update_config_var "$agent_env_file" "NODE_PRIV_KEY" "$NODE_PRIV_KEY"
# bsns-s environment variables
update_config_var "$bsns_s_env_file" "INITIAL_PEER" "$INITIAL_PEER"
update_config_var "$bsns_s_env_file" "NODE_PRIV_KEY" "$NODE_PRIV_KEY"
update_config_var "$bsns_s_env_file" "HUGGINGFACE_API_KEY" "$HUGGINGFACE_API_KEY"
# bsns-c environment variables
update_config_var "$bsns_c_env_file" "PUBLIC_IP" "$PUBLIC_IP"
update_config_var "$bsns_c_env_file" "NODE_PRIV_KEY" "$NODE_PRIV_KEY"
# Orchestrator environment variables
update_config_var "$orchestrator_env_file" "IS_DIST" "$IS_DIST"
update_config_var "$orchestrator_env_file" "HUGGINGFACE_API_KEY" "$HUGGINGFACE_API_KEY"
update_config_var "$orchestrator_env_file" "MONIKER" "$MONIKER"
update_config_var "$orchestrator_env_file" "NESA_NODE_TYPE" "$NESA_NODE_TYPE"
update_config_var "$orchestrator_env_file" "NODE_PRIV_KEY" "$NODE_PRIV_KEY"
# Base environment variables
update_config_var "$base_env_file" "MODEL_NAME" "$MODEL_NAME"
update_config_var "$base_env_file" "MONIKER" "$MONIKER"
update_config_var "$base_env_file" "OP_EMAIL" "$OP_EMAIL"
update_config_var "$base_env_file" "REF_CODE" "$REF_CODE"
update_config_var "$base_env_file" "PUBLIC_IP" "$PUBLIC_IP"
update_config_var "$base_env_file" "ORC_PORT" "$ORC_PORT"
update_config_var "$base_env_file" "NODE_OS" "$NODE_OS"
update_config_var "$base_env_file" "NODE_ARCH" "$NODE_ARCH"
update_config_var "$base_env_file" "NODE_CPU" "$NODE_CPU"
update_config_var "$base_env_file" "NODE_CORES" "$NODE_CORES"
update_config_var "$base_env_file" "NODE_RAM" "$NODE_RAM"
update_config_var "$base_env_file" "NODE_GPU" "$NODE_GPU"
update_config_var "$base_env_file" "NODE_GPU_COUNT" "$NODE_GPU_COUNT"
update_config_var "$base_env_file" "NODE_VRAM" "$NODE_VRAM"
}
display_config() {
local exclude_keys=("HUGGINGFACE_API_KEY" "NODE_PRIV_KEY")
local config_content
local priv_key_display
config_content=$(cat "$config_env_file" "$agent_env_file" "$bsns_c_env_file" "$bsns_s_env_file" "$orchestrator_env_file" "$base_env_file" | sort | uniq)
for key in "${exclude_keys[@]}"; do
config_content=$(echo "$config_content" | grep -v "^$key=")
done
config_content=$(echo "$config_content" | grep -v "=$")
if [[ -n "$NODE_ID" && "$NODE_ID" != "pending..." ]]; then
config_content="$config_content"$'\n'"NODE_ID=$NODE_ID"
fi
if [[ -n "$NODE_PRIV_KEY" ]]; then
local pub_key=$(generate_public_key "$NODE_PRIV_KEY")
local length=${#NODE_PRIV_KEY}
priv_key_display="$(printf '%*s' "$((length - 4))" '' | tr ' ' '*')${NODE_PRIV_KEY: -4}"
config_content="$config_content"$'\n'"PRIVATE_KEY=$priv_key_display\nPUBLIC_KEY=$pub_key"
fi
config_content=$(echo "$config_content" | sed 's/"//g')
config_content="\`\`\`Makefile\n$config_content\n\`\`\`"
echo -e "$config_content" | gum format --type markdown --theme dracula
}
compose_up() {
local compose_files="compose.yml"
local nvidia_present=$(command -v nvidia-smi)
local compose_ext=".yml"
cd "$WORKING_DIRECTORY/docker" || {
echo "Error: Docker directory does not exist."
exit 1
}
if [[ -n "$nvidia_present" ]] && [[ "${NOGPU,,}" != "true" ]] && [[ "${NOGPU,,}" != "1" ]]; then
compose_ext=".nvidia.yml"
fi
if [[ "$NESA_NODE_TYPE" == "community" ]]; then
compose_files="compose.community${compose_ext}"
else
if [[ "$IS_CHAIN" == "yes" ]] || [[ "$IS_VALIDATOR" == "yes" ]]; then
compose_files+=" -f compose.chain.yml"
fi
if [[ "$IS_MINER" == "yes" ]]; then
if [[ "$MINER_TYPE" == "$miner_type_non_distributed" ]]; then
compose_files+=" -f compose.non-dist${compose_ext}"
elif [[ "$MINER_TYPE" == "$miner_type_distributed" ]]; then
if [[ "$DISTRIBUTED_TYPE" == "$distributed_type_new_swarm" ]]; then
compose_files+=" -f compose.bsns-c${compose_ext}"
elif [[ "$DISTRIBUTED_TYPE" == "$distributed_type_existing_swarm" ]]; then
compose_files+=" -f compose.bsns-s${compose_ext}"
fi
fi
fi
fi
docker compose -f $compose_files up --pull always -d --wait
if [[ $? -ne 0 ]]; then
echo "Error: Docker Compose failed to start."
exit 1
else
echo "Docker Compose started successfully."
fi
}
load_node_id() {
if [[ -f "$node_id_file" ]]; then
# Read the value from the file into an environment variable
NODE_ID=$(cat "$node_id_file")
else
# Set the environment variable to an empty string or default value
NODE_ID="pending..."
fi
}
load_from_env_file() {
if [ -f "$config_env_file" ]; then
source "$config_env_file"
elif [ "$1" == "advanced" ]; then
echo "$config_env_file does not exist. Please run in wizard mode to create the config file."
exit 1
else
mkdir -p "$env_dir"
touch "$config_env_file"
fi
if [ -f "$agent_env_file" ]; then
source "$agent_env_file"
elif [ "$1" != "advanced" ]; then
touch "$agent_env_file"
fi
if [ -f "$bsns_s_env_file" ]; then
source "$bsns_s_env_file"
elif [ "$1" != "advanced" ]; then
touch "$bsns_s_env_file"
fi
if [ -f "$bsns_c_env_file" ]; then
source "$bsns_c_env_file"
elif [ "$1" != "advanced" ]; then
touch "$bsns_c_env_file"
fi
if [ -f "$orchestrator_env_file" ]; then
source "$orchestrator_env_file"
elif [ "$1" != "advanced" ]; then
touch "$orchestrator_env_file"
fi
if [ -f "$fluentbit_env_file" ]; then
source "$fluentbit_env_file"
elif [ "$1" != "advanced" ]; then
touch "$fluentbit_env_file"
fi
if [ -f "$base_env_file" ]; then
source "$base_env_file"
elif [ "$1" != "advanced" ]; then
touch "$base_env_file"
fi
# defaults
: ${IS_CHAIN:="no"}
: ${IS_VALIDATOR:="no"}
: ${IS_MINER:="no"}
: ${MINER_TYPE:=$MINER_TYPE_NONE}
: ${DISTRIBUTED_TYPE:=$DISTRIBUTED_TYPE_NONE}
: ${NESA_NODE_TYPE:="community"}
# TODO: revisit below
: ${PRIV_KEY:=""}
: ${HUGGINGFACE_API_KEY:=""}
: ${MODEL_NAME:=""}
: ${REF_CODE:=""}
}
load_from_env_file "wizard"
load_node_id
# don't use cached/saved values for these
PUBLIC_IP=$(curl -s4 ifconfig.me)
#
# bootstrap core logic
#
# deps
check_gum_installed
check_docker_installed
check_jq_installed
check_python_and_ecdsa
# check_nvidia_installed
detect_hardware_capabilities
clear
update_header
echo -e "Select a $(gum style --foreground "$main_color" "mode")"
wizard_mode="Wizardy"
advanced_mode="Advanced Wizardy"
mode=$(gum choose "$wizard_mode" "$advanced_mode")
clear
update_header
gum spin -s line --title "Setting up working directory and cloning node repository..." -- setup_work_dir
setup_work_dir
if grep -q "$advanced_mode" <<<"$mode"; then
load_from_env_file "advanced"
else
MONIKER=$(gum input --cursor.foreground "${main_color}" \
--prompt.foreground "${main_color}" \
--prompt "Choose a moniker for your node: " \
--placeholder "$MONIKER" \
--width 80 \
--value "$MONIKER")
MONIKER=$(echo "$MONIKER" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
clear
update_header
if [[ "$NESA_NODE_TYPE" == "nesa" ]]; then
NODE_HOSTNAME=${NODE_HOSTNAME:-"$MONIKER.yourdomain.tld"}
NODE_HOSTNAME=$(gum input --cursor.foreground "${main_color}" \
--prompt.foreground "${main_color}" \
--prompt "What will $(gum style --foreground "main_color" "$MONIKER")'s hostname be? " \
--placeholder "$NODE_HOSTNAME" \
--width 80 \
--value "$NODE_HOSTNAME")
clear
update_header
else
NODE_HOSTNAME=${NODE_HOSTNAME:-"$MONIKER"}
fi
OP_EMAIL=${OP_EMAIL:-"admin@$NODE_HOSTNAME"}
OP_EMAIL=$(gum input --cursor.foreground "${main_color}" \
--prompt.foreground "${main_color}" \
--prompt "What is the email of the node operator? " \
--placeholder "$OP_EMAIL" \
--width 80 \
--value "$OP_EMAIL")
REF_CODE=$(gum input --cursor.foreground "${main_color}" \
--prompt.foreground "${main_color}" \
--prompt "if you have a referral code, enter it here to receive bonus points: " \
--placeholder "nesa1j6y248qnuawdnd7dtc3hg47jlzfj3jzwqv8rkq" \
--width 160 \
--value "$REF_CODE")
HUGGINGFACE_API_KEY=$(
gum input --cursor.foreground "${main_color}" \
--prompt.foreground "${main_color}" \
--prompt "Please provide your Huggingface API key: " \
--password \
--placeholder "$HUGGINGFACE_API_KEY" \
--width 160 \
--value "$HUGGINGFACE_API_KEY"
)
prompt_for_node_pk=0
if [ -n "$NODE_PRIV_KEY" ]; then
if ! gum confirm "Do you want to use the existing private key? "; then
prompt_for_node_pk=1
fi
else
prompt_for_node_pk=1
fi
if [ "$prompt_for_node_pk" -eq 1 ]; then
NODE_PRIV_KEY=$(gum input --cursor.foreground "${main_color}" \
--password \
--prompt.foreground "${main_color}" \
--prompt "Node's wallet private key: " \
--width 160)
fi
NODE_PRIV_KEY=$(strip_0x_prefix "$NODE_PRIV_KEY")
clear
update_header
if [[ "$NESA_NODE_TYPE" == "nesa" ]]; then
echo -e "Now, what type(s) of node is $(gum style --foreground "$main_color" "$MONIKER")? (use space to select which type(s)"
chain_string="Base"
validator_string="Validator"
miner_string="Miner"
previous_node_type="$chain_string"
if [[ "$IS_CHAIN" == "yes" ]]; then
previous_node_type="$chain_string"
fi
if [[ "$IS_VALIDATOR" == "yes" ]]; then
if [[ -n "$previous_node_type" ]]; then
previous_node_type+=","
fi
previous_node_type+="$validator_string"
fi
if [[ "$IS_MINER" == "yes" ]]; then
if [[ -n "$previous_node_type" ]]; then
previous_node_type+=","
fi
previous_node_type+="$miner_string"
fi
node_type=$(gum choose "$validator_string" "$miner_string" --selected "$previous_node_type")
grep -q "$chain_string" <<<"$node_type" && IS_CHAIN="yes" || IS_CHAIN="no"
grep -q "$validator_string" <<<"$node_type" && IS_VALIDATOR="yes" || IS_VALIDATOR="no"
grep -q "$miner_string" <<<"$node_type" && IS_MINER="yes" || IS_MINER="no"
clear
update_header
if grep -q "$validator_string" <<<"$node_type"; then
echo -e "We are only bootstrapping miner nodes at the moment."
echo -e "Please apply to run a $(gum style --foreground "$main_color" "validator") node here: https://forms.gle/3fQQHVJbHqTPpmy58"
exit 1
# download_import_key_expect
# if [ ! -n "$PRIV_KEY" ]; then
# PRIV_KEY=$(gum input --cursor.foreground "${main_color}" \
# --password \
# --prompt.foreground "${main_color}" \
# --prompt "Validator's private key: " \
# --width 80)
# clear
# update_header
# PASSWORD=$(gum input --cursor.foreground "${main_color}" \
# --password \
# --prompt.foreground "${main_color}" \
# --prompt "Password for the private key: " \
# --width 80)
# docker pull ghcr.io/nesaorg/nesachain/nesachain:test
# docker volume create nesachain-data
# docker run --rm -v nesachain-data:/app/.nesachain -e MONIKER="$MONIKER" -e CHAIN_ID="$CHAIN_ID" -p 26656:26656 -p 26657:26657 -p 1317:1317 -p 9090:9090 -p 2345:2345 $chain_container
# "$WORKING_DIRECTORY/import_key.expect" "$MONIKER" "$PRIV_KEY" "$chain_container" "$PASSWORD"
# fi
# docker run --rm --entrypoint sh -v nesachain-data:/app/.nesachain -p 26656:26656 -p 26657:26657 -p 1317:1317 -p 9090:9090 -p 2345:2345 $chain_container -c '
# VAL_PUB_KEY=$(nesad tendermint show-validator | jq -r ".key") && \
# echo "VAL_PUB_KEY: $VAL_PUB_KEY" && \
# jq -n \
# --arg pubkey "$VAL_PUB_KEY" \
# --arg amount "100000000000unes" \
# --arg moniker "'"$MONIKER"'" \
# --arg chain_id "'"$CHAIN_ID"'" \
# --arg commission_rate "0.10" \
# --arg commission_max_rate "0.20" \
# --arg commission_max_change_rate "0.01" \
# --arg min_self_delegation "1" \
# '"'"'{
# pubkey: {"@type":"/cosmos.crypto.ed25519.PubKey", "key": $pubkey},
# amount: $amount,
# moniker: $moniker,
# "commission-rate": $commission_rate,
# "commission-max-rate": $commission_max_rate,
# "commission-max-change-rate": $commission_max_change_rate,
# "min-self-delegation": $min_self_delegation
# }'"'"' > /app/.nesachain/validator.json && \
# cat /app/.nesachain/validator.json
# '
# docker run --rm --entrypoint nesad -v nesachain-data:/app/.nesachain $chain_container tx staking create-validator /app/.nesachain/validator.json --from "$MONIKER" --chain-id "$CHAIN_ID" --gas auto --gas-adjustment 1.5 --node https://rpc.test.nesa.ai
fi
if grep -q "$miner_string" <<<"$node_type"; then
clear
update_header
echo -e "Now, what type of miner will $(gum style --foreground "$main_color" "$MONIKER") be?"
distributed_string="Distributed Miner"
non_distributed_string="Non-Distributed Miner"
if [[ "$MINER_TYPE" == "$miner_type_distributed" ]]; then
default_miner_choice="$distributed_string"
else
default_miner_choice="$non_distributed_string"
fi
selected_miner_type=$(gum choose "$distributed_string" "$non_distributed_string" --selected "$default_miner_choice")
clear
update_header
if grep -q "$selected_miner_type" <<<"$distributed_string"; then
IS_DIST=True # TODO: update containers to rely on DISTRIBUTED_TYPE instead of IS_DIST
MINER_TYPE=$miner_type_distributed
echo -e "Would you like to join an existing $(gum style --foreground "$main_color" "swarm") or start a new one?"
existing_swarm="Join existing swarm"
new_swarm="Start a new swarm"
if [[ "$DISTRIBUTED_TYPE" == "$distributed_type_new_swarm" ]]; then
default_swarm_choice="$new_swarm"
else
default_swarm_choice="$existing_swarm"
fi
selected_distributed_type=$(gum choose "$existing_swarm" "$new_swarm" --selected "$default_swarm_choice")
clear
update_header
if grep -q "$selected_distributed_type" <<<"$new_swarm"; then
DISTRIBUTED_TYPE=$distributed_type_new_swarm
MODEL_NAME=$(
gum input --cursor.foreground "${main_color}" \
--prompt.foreground "${main_color}" \
--prompt "Which model would you like to run? " \
--placeholder "$MODEL_NAME" \
--width 80 \
--value "$MODEL_NAME"
)
else
DISTRIBUTED_TYPE=$distributed_type_existing_swarm
swarms_map=$(get_swarms_map)
model_names=$(get_model_names "$swarms_map")
echo -e "Which existing $(gum style --foreground "$main_color" "swarm") would you like to join?"
MODEL_NAME=$(echo "$model_names" | gum choose)
initial_peer_id=$(get_node_id "$swarms_map" "$MODEL_NAME")
node_lookup_id=$(create_combined_node_id "$swarms_map" "$MODEL_NAME")
initial_peer_ip=$(fetch_network_address "$node_lookup_id")
INITIAL_PEER="/ip4/$initial_peer_ip/tcp/31330/p2p/$initial_peer_id"
fi
else
MINER_TYPE=$miner_type_non_distributed
DISTRIBUTED_TYPE=$distributed_type_none
IS_DIST=False # deprecrated: update containers to rely on DISTRIBUTED_TYPE instead of IS_DIST
MODEL_NAME=$(
gum input --cursor.foreground "${main_color}" \
--prompt.foreground "${main_color}" \
--prompt "Which model would you like to run? " \
--placeholder "nlptown/bert-base-multilingual-uncased-sentiment" \
--width 160 \
--value "$MODEL_NAME"
)
fi
clear
update_header
fi
else
MINER_TYPE=$miner_type_agnostic
DISTRIBUTED_TYPE=$distributed_type_agnostic
IS_MINER="yes"
IS_DIST=False
fi
fi
save_to_env_file
clear
update_header
display_config
if ! gum confirm "Do you want to start the node with the above configuration? "; then
echo "Configuration saved. You can modify the configuration manually, run the wizard again, or you can simply use advanced wizardry to boot your node."
exit 0
fi
cd "$WORKING_DIRECTORY/docker" || {
echo "Error: Docker directory does not exist."
exit 1
}
compose_up
cd "$init_pwd" || return
echo -e "Congratulations! Your $(gum style --foreground "$main_color" "nesa") node was successfully bootstrapped!"
# set +x