#!/usr/bin/env python3 # This script will use the libcloud api # to manage vps instances from # 1) Google Cloud Platform (GCE) # 2) Microsoft Azure (AZURE_ARM) # 3) Amazon Web Services (EC2) # A user will be able to # 1) create an instance # 2) delete an instance # 3) list all instances across all cloud providers # 4) start a stopped instance # 5) stop a running instance # 6) reboot a running instance # 7) list images available # 8) list sizes available # 9) run a script on a newly created instance # from the command line using flags import os import re import sys import argparse import time import json import subprocess import libcloud.security import inspect import paramiko from dotenv import load_dotenv from libcloud.compute.types import Provider from libcloud.compute.providers import get_driver from libcloud.compute.base import NodeAuthSSHKey from libcloud.compute.deployment import ScriptDeployment # Disable SSL certificate verification # Disable SHA-2 variants of RSA key verification algorithm for backward compatibility reasons # Declare the ENV_FILE variable as such to always reside in the same directory as the script # We use os.path.join to make sure the path is correct for every OS ENV_FILE = os.path.join(os.path.dirname(__file__), ".env") SECDEP_SSH_PUBLIC_KEY = os.path.join(os.path.dirname(__file__), "secdep.pub") SECDEP_SSH_PRIVATE_KEY = os.path.join(os.path.dirname(__file__), "secdep") if not len(sys.argv) > 1: print("No arguments passed. Use -h or --help for help") exit(0) # Define the command line arguments parser = argparse.ArgumentParser( prog='secdep.py', description='Manage cloud instances', ) parser.add_argument('-l', '--list', help='List all instances', action='store_true') parser.add_argument('-v', '--values', help='Change credential values', action='store_true') parser.add_argument('-P', '--provider', help='Cloud provider', choices=['gce', 'azure', 'aws']) parser.add_argument('-a', '--action', help='Action to perform', choices=['create', 'delete', 'start', 'stop', 'reboot']) parser.add_argument('-c', '--create', help='Create an instance', action='store_true') parser.add_argument('-d', '--delete', help='Delete an instance', action='store_true') parser.add_argument('-x', '--start', help='Start an instance', action='store_true') parser.add_argument('-z', '--stop', help='Stop an instance', action='store_true') parser.add_argument('-r', '--reboot', help='Reboot an instance', action='store_true') parser.add_argument('-f', '--file', help='File to run on an instance', type=str, default=None, required=False) parser.add_argument('-p', '--listimages', help='List images', action='store_true') parser.add_argument('-q', '--listsizes', help='List sizes', action='store_true') parser.add_argument('-G', '--listlocations', help='List locations', action='store_true') parser.add_argument('-i', '--image', help='Image to use') parser.add_argument('-s', '--size', help='Size of instance') parser.add_argument('-n', '--name', help='Name of instance') parser.add_argument('-g', '--region', help='Region to use') parser.add_argument('-y', '--yes', help='Do not ask for confirmation', action='store_true') parser.add_argument('-k', '--print', help='Also print image, location or size', action='store_true') args = parser.parse_args() if not os.path.exists(SECDEP_SSH_PUBLIC_KEY) or not os.path.exists(SECDEP_SSH_PRIVATE_KEY): # Generate a new SSH key pair # The key is stored in the current directory and named secdep # The public key is stored in the current directory and named secdep.pub # The passphrase is an empty string # The key is a 4096 bit RSA key key = paramiko.RSAKey.generate(4096) key.write_private_key_file(SECDEP_SSH_PRIVATE_KEY) with open(SECDEP_SSH_PUBLIC_KEY, 'w') as f: f.write("%s %s" % (key.get_name(), key.get_base64())) # We first check if it exists already in order to avoid overwriting it # and if it doesn't exist, we create it by inserting an empty string # When using the with statement, the file is automatically # closed at the end of the indented block if not os.path.exists(ENV_FILE): with open(ENV_FILE, 'w') as f: f.write('') # The required values for authentication are stored in the .env file # These are # 1) SECDEP_GCE_CLIENT_ID (the service account Email found in project's IAM & Admin section/Service Accounts) # 2) SECDEP_GCE_CLIENT_SECRET (the service account's private Key ID found in project's IAM & Admin section/Service Accounts) # 3) SECDEP_GCE_PROJECT_ID (the project ID found in project's dashboard) # 4) SECDEP_AZURE_TENANT_ID # 5) SECDEP_AZURE_SUB_ID # 6) SECDEP_AZURE_APP_ID # 7) SECDEP_AZURE_PASSWORD # 8) SECDEP_AWS_ACCESS_KEY # 9) SECDEP_AWS_SECRET_KEY # For GCE we need to create a service account (with Owner Role from the IAM section) and download the json file (from # the Service Account's manage keys section) in the same directory as the script # We then check if the .env file is empty if os.stat(ENV_FILE).st_size == 0: print('You will be asked for each needed value\nIf you want to skip a provider press enter on each of their values because they are all needed for authentication\nIf at some point you delete the provider\'s value entry you will once again be asked to enter it\nIf you pressed enter by mistake or inserted an incorrect value just edit the file directly or delete the corresponding line\nThere is also the choice of using the -v option to have that done interactively') # We search for these values in the ENV_FILE and for each not found, we prompt the user to enter it # We then write the values to the ENV_FILE # An empty string is allowed in case the user does not want to use a particular cloud provider # but he will be prompted for all values with open(ENV_FILE, 'r') as f: env_file_content = f.read() if 'SECDEP_GCE_CLIENT_ID' not in env_file_content: SECDEP_GCE_CLIENT_ID = input("Enter your GCE_CLIENT_ID: ") with open(ENV_FILE, 'a') as f: f.write('SECDEP_GCE_CLIENT_ID={}\n'.format(SECDEP_GCE_CLIENT_ID)) if 'SECDEP_GCE_CLIENT_SECRET' not in env_file_content: SECDEP_GCE_CLIENT_SECRET = input("Enter your GCE_CLIENT_SECRET: ") with open(ENV_FILE, 'a') as f: f.write('SECDEP_GCE_CLIENT_SECRET={}\n'.format(SECDEP_GCE_CLIENT_SECRET)) if 'SECDEP_GCE_PROJECT_ID' not in env_file_content: SECDEP_GCE_PROJECT_ID = input("Enter your GCE_PROJECT_ID: ") with open(ENV_FILE, 'a') as f: f.write('SECDEP_GCE_PROJECT_ID={}\n'.format(SECDEP_GCE_PROJECT_ID)) if 'SECDEP_AZURE_TENANT_ID' not in env_file_content: SECDEP_AZURE_TENANT_ID = input("Enter your AZURE_TENANT_ID: ") with open(ENV_FILE, 'a') as f: f.write('SECDEP_AZURE_TENANT_ID={}\n'.format(SECDEP_AZURE_TENANT_ID)) if 'SECDEP_AZURE_SUB_ID' not in env_file_content: SECDEP_AZURE_SUB_ID = input("Enter your AZURE_SUB_ID: ") with open(ENV_FILE, 'a') as f: f.write('SECDEP_AZURE_SUB_ID={}\n'.format(SECDEP_AZURE_SUB_ID)) if 'SECDEP_AZURE_APP_ID' not in env_file_content: SECDEP_AZURE_APP_ID = input("Enter your AZURE_APP_ID: ") with open(ENV_FILE, 'a') as f: f.write('SECDEP_AZURE_APP_ID={}\n'.format(SECDEP_AZURE_APP_ID)) if 'SECDEP_AZURE_PASSWORD' not in env_file_content: SECDEP_AZURE_PASSWORD = input("Enter your AZURE_PASSWORD: ") with open(ENV_FILE, 'a') as f: f.write('SECDEP_AZURE_PASSWORD={}\n'.format(SECDEP_AZURE_PASSWORD)) if 'SECDEP_AWS_ACCESS_KEY' not in env_file_content: SECDEP_AWS_ACCESS_KEY = input("Enter your AWS_ACCESS_KEY: ") with open(ENV_FILE, 'a') as f: f.write('SECDEP_AWS_ACCESS_KEY={}\n'.format(SECDEP_AWS_ACCESS_KEY)) if 'SECDEP_AWS_SECRET_KEY' not in env_file_content: SECDEP_AWS_SECRET_KEY = input("Enter your AWS_SECRET_KEY: ") with open(ENV_FILE, 'a') as f: f.write('SECDEP_AWS_SECRET_KEY={}\n'.format(SECDEP_AWS_SECRET_KEY)) # Load environment variables from ENV_FILE and temporarily store them # along the other system environment variables # If an environment variable is already set, it will not be overwritten # but since the existence of the .env file is mandatory it is best if # they are not already set. # That is because some of them may be named something slightly different # by the user and this program works with these exact value names # That is also the reason they are prefixed with the SECDEP_ string load_dotenv(ENV_FILE) # We also store them in variables for ease of use inside the program # We take for granted that these values exist either empty or not def get_env_vars(): # Make them global so we can use them in other functions global SECDEP_GCE_CLIENT_ID global SECDEP_GCE_CLIENT_SECRET global SECDEP_GCE_PROJECT_ID global SECDEP_AZURE_TENANT_ID global SECDEP_AZURE_SUB_ID global SECDEP_AZURE_APP_ID global SECDEP_AZURE_PASSWORD global SECDEP_AWS_ACCESS_KEY global SECDEP_AWS_SECRET_KEY # GCE SECDEP_GCE_CLIENT_ID = os.getenv('SECDEP_GCE_CLIENT_ID') SECDEP_GCE_CLIENT_SECRET = os.getenv('SECDEP_GCE_CLIENT_SECRET') SECDEP_GCE_PROJECT_ID = os.getenv('SECDEP_GCE_PROJECT_ID') if SECDEP_GCE_CLIENT_SECRET !="" and SECDEP_GCE_PROJECT_ID !="": SECDEP_GCE_CLIENT_SECRET = os.path.join(os.path.dirname(os.path.abspath(__file__)), SECDEP_GCE_PROJECT_ID+'-'+SECDEP_GCE_CLIENT_SECRET[:12]+'.json') # Never mind the error after converting is not None to !="" because only then # this variable doesnt end up as /home/konsthol/MyGitea/SecDep/-.json # Azure SECDEP_AZURE_TENANT_ID = os.getenv('SECDEP_AZURE_TENANT_ID') SECDEP_AZURE_SUB_ID = os.getenv('SECDEP_AZURE_SUB_ID') SECDEP_AZURE_APP_ID = os.getenv('SECDEP_AZURE_APP_ID') SECDEP_AZURE_PASSWORD = os.getenv('SECDEP_AZURE_PASSWORD') # AWS SECDEP_AWS_ACCESS_KEY = os.getenv('SECDEP_AWS_ACCESS_KEY') SECDEP_AWS_SECRET_KEY = os.getenv('SECDEP_AWS_SECRET_KEY') get_env_vars() # List all the entries in the .env file and ask the user which one he wants to update # If the user enters an invalid value, the program will ask again # If the user enters 0 the program will exit def update_env_file(): with open(ENV_FILE, 'r') as f: env_file_content = f.read() # find all lines staring with SECDEP_ file_entries = list(filter(lambda x: x.startswith('SECDEP_'), env_file_content.split('\n'))) count = 0 for line in file_entries: count += 1 print("{}) {}".format(count, line)) print("Choosing 0 will exit the function") print("You will be asked to enter the new value until it is valid or you enter 0") choice = input("Choose the entry you want to update: ") try: choice = int(choice) if choice > count or choice < 0: raise ValueError elif choice == 0: return except ValueError: print("Invalid choice") update_env_file() else: entry = file_entries[choice - 1] entry_name = entry.split('=')[0] entry_value = entry.split('=')[1] if entry_value == '': entry_value = 'None' print("The current value for {} is {}".format(entry_name, entry_value)) new_value = input("Enter the new value: ") with open(ENV_FILE, 'w') as f: f.write(env_file_content.replace(entry, "{}={}".format(entry_name, new_value))) print("The value for {} was updated successfully".format(entry_name)) update_env_file() # Reload the environment variables load_dotenv(ENV_FILE) get_env_vars() AWS_ubuntu22_04_images = { # "af-south-1": "ami-0e464b219a424deaf", # "ap-east-1": "ami-0bc44b8dc7cae9c34", "ap-northeast-1": "ami-0cd7ad8676931d727", "ap-south-1": "ami-06984ea821ac0a879", "ca-central-1": "ami-0dae3a932d090b3de", "eu-central-1": "ami-03e08697c325f02ab", "eu-north-1": "ami-00c70b245f5354c0a", # "eu-south-1": "ami-04f5f6ddf256cb634", "eu-west-1": "ami-0333305f9719618c7", # "me-central-1": "ami-0242106c866413fc8", # "me-south-1": "ami-0a1fdc43b81c7158b", "sa-east-1": "ami-015e30624fffff117", "us-east-1": "ami-00874d747dde814fa", "us-west-1": "ami-09b2a1e33ce552e68", # "us-gov-east-1": "ami-0efd49eddc5639cc5", # "us-gov-west-1": "ami-061efa908c09c5409", "ap-northeast-2": "ami-09eba584c30b7299f", # "ap-south-2": "ami-063b634d62a631728", "ap-southeast-2": "ami-0c18f3cdeea1c220d", # "eu-central-2": "ami-093264efdd8abd77a", # "eu-south-2": "ami-0e2cc458bd7bcef78", "eu-west-2": "ami-0d09654d0a20d3ae2", "us-east-2": "ami-0ab0629dba5ae551d", "us-west-2": "ami-095413544ce52437d", "ap-northeast-3": "ami-0ead712799090892e", # "ap-southeast-3": "ami-093bbf368c0fe38fe", "eu-west-3": "ami-0afd55c0c8a52973a" } AWS_ubuntu22_10_images = { # "af-south-1": "ami-0eb79b9557c227d8b", # "ap-east-1": "ami-0649ce955b9ae6a6c", "ap-northeast-1": "ami-04e096919731466e4", "ap-south-1": "ami-01b60efb0632e8b18", "ap-southeast-1": "ami-085ecfe4d567e153a", "ca-central-1": "ami-010346a896f698f7d", "eu-central-1": "ami-0f1cf34dcb4057a5f", "eu-north-1": "ami-0e942149c97ff88bd", # "eu-south-1": "ami-001081dd6e3c2cd7f", "eu-west-1": "ami-015423a987dafce81", # "me-central-1": "ami-06e1d0e6cc8c23f9e", # "me-south-1": "ami-0ceb0cbe999a08ab4", "sa-east-1": "ami-06edb0fb80530aa8b", "us-east-1": "ami-074f5b0bdbdc43210", "us-west-1": "ami-0eb5f373317fb0ae4", "ap-northeast-2": "ami-05493e234687bd083", # "ap-south-2": "ami-0f672f06263d4cac9", "ap-southeast-2": "ami-0bd29331dcb023a43", # "eu-central-2": "ami-068fda3bce227e2ed", # "eu-south-2": "ami-03f876aeda1da1eba", "eu-west-2": "ami-0ae7948375d66d902", "us-east-2": "ami-0de381f43eb6dbe39", "us-west-2": "ami-0534f435d9dd0ece4", "ap-northeast-3": "ami-05ede9a3a46934154", # "ap-southeast-3": "ami-00b6bff126cff8ea4", "eu-west-3": "ami-0545575c963ed1276" } AWS_debian_10_images = { # "af-south-1": "ami-0fca764f342bb4e75", # "ap-east-1": "ami-0e1391d32ca6645e5", "ap-northeast-1": "ami-041e370f47f9b619e", "ap-northeast-2": "ami-0243bb2bec2d9707b", "ap-northeast-3": "ami-0c1c0379b3534e7d8", "ap-south-1": "ami-0eb2c4104acb437b2", # "ap-south-2": "ami-0b87809e702026b3d", "ap-southeast-1": "ami-0ee39036464b9a87e", "ap-southeast-2": "ami-0bb99834ae8c7471e", # "ap-southeast-3": "ami-051fb80fedf8626d4", "ca-central-1": "ami-0125327537ef16282", "eu-central-1": "ami-0c984d7a384cafb51", # "eu-central-2": "ami-05cfba925613cba71", "eu-north-1": "ami-013fff3fbe1b44cd7", # "eu-south-1": "ami-076d9ee33498e5425", # "eu-south-2": "ami-0a9bd5a675e804f5b", "eu-west-1": "ami-00aa3f69e07141166", "eu-west-2": "ami-073e61682f2137943", "eu-west-3": "ami-0fa84148519465f59", # "me-central-1": "ami-0e6d2fa064807e5a1", # "me-south-1": "ami-0166968d754514d17", "sa-east-1": "ami-03405dcb7a98e974b", "us-east-1": "ami-03d6e2aceefaa35b0", "us-east-2": "ami-0246e87085c5c98e3", "us-west-1": "ami-0809b44a732f37188", "us-west-2": "ami-0164ab05efc075cbc" } AWS_debian_11_images = { # "af-south-1": "ami-01d91bb5413549aee", # "ap-east-1": "ami-04162e945020a0a90", "ap-northeast-1": "ami-043fed2cdcba4027e", "ap-northeast-2": "ami-0bb0231c184073ac1", "ap-northeast-3": "ami-09968ae34a843a6cb", "ap-south-1": "ami-079b117c1800d30f8", # "ap-south-2": "ami-039abd0992b9b40bf", "ap-southeast-1": "ami-07ac0a74d21a3174c", "ap-southeast-2": "ami-0c7a13e86ce7afc80", # "ap-southeast-3": "ami-02acfafb90c8fc9e3", "ca-central-1": "ami-09b3e61273dd34f3b", "eu-central-1": "ami-0c75b861029de4030", # "eu-central-2": "ami-09ff43dae49ca4766", "eu-north-1": "ami-08869bacfa1188ec9", # "eu-south-1": "ami-058bf36d271338447", # "eu-south-2": "ami-0401e98c615a7854e", "eu-west-1": "ami-0591c8c8aa7d9b217", "eu-west-2": "ami-0e789c3dd24faa0be", "eu-west-3": "ami-040dc155c278da35a", # "me-central-1": "ami-0b5a9e952bc027304", # "me-south-1": "ami-058869731da985c55", "sa-east-1": "ami-0cb45622734c55ed6", "us-east-1": "ami-052465340e6b59fc0", "us-east-2": "ami-06a7641d5bd7bdc65", "us-west-1": "ami-0097d5326aebc68e0", "us-west-2": "ami-05063446e767da4ff" } AWS_centos7_images = { "eu-north-1": "ami-0e5125a0f19c52a2b", "eu-west-2": "ami-0de2f45684e59282c", # "af-south-1": "ami-0cf2f235226a4d1f2", "us-west-1": "ami-0bcd12d19d926f8e9", "eu-central-1": "ami-0afcbcee3dfbce929", "us-east-1": "ami-0aedf6b1cb669b4c7", # "eu-south-1": "ami-0a3a6d4d737db3bc1", "ap-south-1": "ami-09f129ee53d3523c0", # "me-south-1": "ami-09ce1cd69cab28540", "ap-northeast-1": "ami-06e6d2122baa563c4", "ap-northeast-2": "ami-061f5322ab2662c82", "eu-west-3": "ami-051806c39fa542e22", "us-west-2": "ami-04f798ca92cc13f74", "ap-southeast-1": "ami-03bfba2e75432064e", # "ap-east-1": "ami-039ac400e94785a1d", "us-east-2": "ami-033adaf0b583374d4", "ap-southeast-2": "ami-0264ead5294ad1773", "ca-central-1": "ami-01ebef6e00efb2c20", "sa-east-1": "ami-015f6bf0657816a2d", "eu-west-1": "ami-00d464afa64e1fc69" } AWS_centos8_images = { "sa-east-1": "ami-0ec17e5479978d435", "ap-southeast-2": "ami-0cd15beeae86bd0b5", "us-east-1": "ami-0c07df890a618c98a", "eu-central-1": "ami-0b79da023fb461a12", "us-west-2": "ami-09195cb76ab892888", # "eu-south-1": "ami-089ad68ee518459be", "eu-west-1": "ami-0819edf1cd94e83e8", "ap-southeast-1": "ami-07ab649c51b1c14e7", "eu-north-1": "ami-05eaebdafff627949", "us-east-2": "ami-05cefb3ebaddc75f6", "eu-west-3": "ami-05a3b9ccef6b9a4f4", # "af-south-1": "ami-05603d0aec38099d3", "ap-south-1": "ami-04ebf66b7be0500f6", "eu-west-2": "ami-0495083a5dc6bc6a3", # "me-south-1": "ami-040740782158cb0e3", # "ap-east-1": "ami-03fa27f9c9dd340fc", "us-west-1": "ami-033e1fe6304139f4c", "ap-northeast-2": "ami-014911a31aba94953", "ap-northeast-1": "ami-0117f2fc9c6939327", "ca-central-1": "ami-00723d0970fc53863" } AWS_centos9_images = { # "eu-south-1": "ami-0f5511636b455063e", "us-west-2": "ami-0e27c3746a0d5ecbf", "sa-east-1": "ami-0c70afd15bcb126a9", # "af-south-1": "ami-0b0c19981dc5e5f1c", "eu-north-1": "ami-0a942e3eea51f9810", # "ap-east-1": "ami-0a8e37e83bdd06c65", "ap-southeast-2": "ami-09825374904820651", "us-west-1": "ami-08e31531a8310da78", "eu-west-1": "ami-08be94d0e177189de", "eu-west-3": "ami-082f57b7c23b057df", "ap-northeast-1": "ami-074800b0d58e64b24", # "me-south-1": "ami-070523667a38e853f", "ap-southeast-1": "ami-05a6e3d785f747c7d", "eu-west-2": "ami-056117c19265dad25", "ca-central-1": "ami-03f6ca47d93885ab9", "ap-south-1": "ami-0383b9760ad3de192", "us-east-1": "ami-027a785419f41ea0f", "ap-northeast-2": "ami-025fab85b691896e8", "us-east-2": "ami-0127d4ce241322a8e", "eu-central-1": "ami-00cdc2b0769957e1c" } AWS_fedora37_images = { "us-east-1": "ami-023fb534213ca41da", "us-east-2": "ami-0d9ef71e892f861c6", "us-west-2": "ami-019b893191a9ac44a", "us-west-1": "ami-04e37334d4e907fab", "eu-west-1": "ami-0f242738c5379ffc1", "eu-central-1": "ami-0965c162c412da7ca", "eu-west-2": "ami-0257e646e0a4b4987", "ap-southeast-1": "ami-002185f8a7d5528e2", "ap-northeast-1": "ami-0e010569d0798fa05", "ap-southeast-2": "ami-028fb965bf530cd3e", "sa-east-1": "ami-0390b84962fd284f8", "ap-northeast-2": "ami-0902b5ee3ed4ebe17", "ap-south-1": "ami-04e6a16b463b1dbe0", "ca-central-1": "ami-057f927818ad393c5" } AWS_redhat9_images = { "eu-north-1": "ami-0cd776c8201793f81", "ap-south-1": "ami-063d0d3250553f017", "eu-west-3": "ami-0b6b7402cc4708282", "eu-west-2": "ami-03628db51da52eeaa", "eu-west-1": "ami-028f9616b17ba1d53", "ap-northeast-3": "ami-088c289c3f07fa115", "ap-northeast-2": "ami-000847072b0b35502", "ap-northeast-1": "ami-0b74caa65e36e4b38", "sa-east-1": "ami-03f8004d2ba700e6c", "ca-central-1": "ami-0c23a35603e3d8ba4", "ap-southeast-1": "ami-07a79552200256856", "ap-southeast-2": "ami-0fe791c0ace58322c", "eu-central-1": "ami-025d24108be0a614c", "us-east-1": "ami-0c41531b8d18cc72b", "us-east-2": "ami-078cbc4c2d057c244", "us-west-1": "ami-0fa0ed170a59f4917", "us-west-2": "ami-04a616933df665b44" } AWS_redhat8_6_images = { "eu-north-1": "ami-06a2a41d455060f8b", "ap-south-1": "ami-05c8ca4485f8b138a", "eu-west-3": "ami-0460bf124812bebfa", "eu-west-2": "ami-035c5dc086849b5de", "eu-west-1": "ami-0f0f1c02e5e4d9d9f", "ap-northeast-3": "ami-044921b7897a7e0da", "ap-northeast-2": "ami-06c568b08b5a431d5", "ap-northeast-1": "ami-0f903fb156f24adbf", "sa-east-1": "ami-0c1b8b886626f940c", "ca-central-1": "ami-0c3d3a230b9668c02", "ap-southeast-1": "ami-051f0947e420652a9", "ap-southeast-2": "ami-0808460885ff81045", "eu-central-1": "ami-0e7e134863fac4946", "us-east-1": "ami-06640050dc3f556bb", "us-east-2": "ami-092b43193629811af", "us-west-1": "ami-0186e3fec9b0283ee", "us-west-2": "ami-08970fb2e5767e3b8" } AWS_redhat7_9_images = { "eu-north-1": "ami-003fb5b0ea327060c", "ap-south-1": "ami-0b6d1128312a13b2a", "eu-west-3": "ami-0f4643887b8afe9e2", "eu-west-2": "ami-0e6c172f77df9f9c3", "eu-west-1": "ami-020e14de09d1866b4", "ap-northeast-3": "ami-00718a107dacde79f", "ap-northeast-2": "ami-0c851e892c33af909", "ap-northeast-1": "ami-0155fdd0956a0c7a0", "sa-east-1": "ami-07eca9d9caaced495", "ca-central-1": "ami-0de9a412a63b8f99d", "ap-southeast-1": "ami-0f24fbd3cc8531844", "ap-southeast-2": "ami-0fb87e863747a1610", "eu-central-1": "ami-0f58468b80db2db66", "us-east-1": "ami-005b7876121b7244d", "us-east-2": "ami-0d2bf41df19c4aac7", "us-west-1": "ami-015474e24281c803d", "us-west-2": "ami-02d40d11bb3aaf3e5" } AWS_opensuseLeap15_3_images = { "ap-south-1": "ami-0ade60d62f2a2e05c", "eu-north-1": "ami-0ae983da6ffb9b1e7", "eu-west-3": "ami-025ea6b220205eeda", "eu-west-2": "ami-0c912bbb879286143", "eu-west-1": "ami-0397bb96c927b3747", "ap-northeast-3": "ami-073fffb8542446f02", "ap-northeast-2": "ami-0a4130f336deb5d7d", "ap-northeast-1": "ami-093be1d241fe4f400", "ca-central-1": "ami-07b73495c9189e14c", "sa-east-1": "ami-02f22e7a3a61b9ad4", "ap-southeast-1": "ami-0e2676834fdb2722d", "ap-southeast-2": "ami-0d75289309da084dd", "eu-central-1": "ami-0ca8f0b74d5fe4b0c", "us-east-1": "ami-0a21376f8d02ffa2d", "us-east-2": "ami-0a3fdbf842659345d", "us-west-1": "ami-0feb167c171099332", "us-west-2": "ami-0eb50cf5b36982ab5", } AWS_opensuseLeap15_4_images = { "ap-south-1": "ami-0c936741ae62acefd", "eu-north-1": "ami-06f677915bcf57a3f", "eu-west-3": "ami-0fcb2baaff359470d", "eu-west-2": "ami-0d75aefcfe115e1b1", "eu-west-1": "ami-04c349804a22e1a81", "ap-northeast-3": "ami-0ce8958d99aded398", "ap-northeast-2": "ami-0d6202fde5a0672cb", "ap-northeast-1": "ami-0ad77d12fa9be7d26", "ca-central-1": "ami-08d3a675ff259e79b", "sa-east-1": "ami-05c679febf019e22b", "ap-southeast-1": "ami-0eff8b51447b310c8", "ap-southeast-2": "ami-0050fc1b597e7e5c7", "eu-central-1": "ami-04050bb7554762bec", "us-east-1": "ami-08159a7e0bbb30b75", "us-east-2": "ami-028019dea1a954aa4", "us-west-1": "ami-0685f2266425e1dec", "us-west-2": "ami-02881e3769e7c9ab2" } AWS_images = { "Ubuntu Server 22.04 LTS": AWS_ubuntu22_04_images, "Ubuntu Server 22.10": AWS_ubuntu22_10_images, "Debian 10": AWS_debian_10_images, "Debian 11": AWS_debian_11_images, "CentOS 7": AWS_centos7_images, "CentOS 8": AWS_centos8_images, "CentOS 9": AWS_centos9_images, "Fedora 37": AWS_fedora37_images, "Red Hat Enterprise Linux 7.9": AWS_redhat7_9_images, "Red Hat Enterprise Linux 8.6": AWS_redhat8_6_images, "Red Hat Enterprise Linux 9.0": AWS_redhat9_images, "OpenSUSE Leap 15.3": AWS_opensuseLeap15_3_images, "OpenSUSE Leap 15.4": AWS_opensuseLeap15_4_images } AZURE_images = { "Ubuntu Server 22.04 LTS": "Canonical:0001-com-ubuntu-server-jammy:22_04-lts:22.04.202301050", "Ubuntu Server 22.10": "Canonical:0001-com-ubuntu-server-kinetic:22_10:22.10.202301040", "Debian 10": "Debian:debian-11:11:0.20221219.1234", "Debian 11": "Debian:debian-10:10:0.20221205.1220", "CentOS 8.4": "OpenLogic:CentOS:8_4:8.4.2021071900", "CentOS 8.5": "OpenLogic:CentOS:8_5:8.5.2022101800", "Fedora 36": "ntegralinc1586961136942:ntg_fedora_36:ntg_fedora_36:1.0.1", "Fedora 37": "ntegralinc1586961136942:ntg_fedora_37:ntg_fedora_37:1.0.0", "Red Hat Enterprise Linux 8.6": "RedHat:rhel-raw:8_6:8.6.2022052401", "Red Hat Enterprise Linux 9.1": "RedHat:rhel-raw:9_1:9.1.2022112213", "OpenSUSE Leap 15.3": "SUSE:opensuse-leap-15-3:gen1:2022.11.04", "OpenSUSE Leap 15.4": "SUSE:opensuse-leap-15-4:gen1:2022.11.04" } global gce_driver global azure_driver global aws_driver # Get GCE driver def get_gce_driver(): if SECDEP_GCE_CLIENT_SECRET !="" and SECDEP_GCE_PROJECT_ID !="" and SECDEP_GCE_CLIENT_ID !="": driver = get_driver(Provider.GCE) print("Trying to authenticate with google...\n") return driver(SECDEP_GCE_CLIENT_ID, SECDEP_GCE_CLIENT_SECRET, project=SECDEP_GCE_PROJECT_ID) # Get Azure driver def get_azure_driver(): if SECDEP_AZURE_TENANT_ID !="" and SECDEP_AZURE_SUB_ID !="" and SECDEP_AZURE_APP_ID !="" and SECDEP_AZURE_PASSWORD !="": driver = get_driver(Provider.AZURE_ARM) print("Trying to authenticate with azure...\n") return driver(tenant_id=SECDEP_AZURE_TENANT_ID, subscription_id=SECDEP_AZURE_SUB_ID, key=SECDEP_AZURE_APP_ID, secret=SECDEP_AZURE_PASSWORD) # Get AWS driver def get_aws_driver(): if SECDEP_AWS_ACCESS_KEY !="" and SECDEP_AWS_SECRET_KEY !="": driver = get_driver(Provider.EC2) print("Trying to authenticate with amazon...\n") return driver(SECDEP_AWS_ACCESS_KEY, SECDEP_AWS_SECRET_KEY) gce_driver = get_gce_driver() azure_driver = get_azure_driver() aws_driver = get_aws_driver() def get_corresponding_driver(provider): global driver match provider: case "gce": driver = gce_driver case "azure": driver = azure_driver case "aws": driver = aws_driver case _: print("Invalid provider") assert driver is not None, "You need to set all {} environment variables first".format(provider.upper()) return driver def list_provider_sizes(provider): print("Getting "+provider+" sizes...") driver = get_corresponding_driver(provider) if provider == "aws" or provider == "gce": sizes = driver.list_sizes(location=None) sizes = [size for size in sizes if size.ram < 16384] else: azlocation = driver.list_locations()[0] sizes = driver.list_sizes(location=azlocation) sizes = [size for size in sizes if size.ram < 16384] count = 0 print("Available "+provider+" sizes") if provider == "aws": for size in sizes: count += 1 print("{}) {}\n\nRam: {}\nDisk: {}\nBandwidth: {}\nPrice: {}\n".format(count, size.name, size.ram, size.disk, size.bandwidth, size.price)) elif provider == "gce": for size in sizes: count += 1 print("{}) {}\n\n{}\nPrice: {}\n".format(count, size.name, size.extra['description'], size.price)) else: for size in sizes: count += 1 print("{}) {}\n\nRam: {}\nDisk: {}\nPrice: {}\n".format(count, size.name, size.ram, size.disk, size.price)) return sizes def list_provider_locations(provider): print("Getting "+provider+" locations...") driver = get_corresponding_driver(provider) locations = driver.list_locations() count = 0 print("Available "+provider+" locations") if provider == "aws": locations = [] awsLocations = ["ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2"] for region in awsLocations: awsdr = get_driver(Provider.EC2)(SECDEP_AWS_ACCESS_KEY, SECDEP_AWS_SECRET_KEY,region=region) specificAwsLocations = awsdr.list_locations() for item in specificAwsLocations: locations.append(item) for location in locations: count += 1 print("{}) {}\n\nRegion name: {}\nCountry: {}\n".format(count, location.name, location.availability_zone.region_name, location.country)) else: for location in locations: count += 1 print("{}) {}\n\nCountry: {}\n".format(count, location.name, location.country)) return locations def listAWSregions(list): count = 0 print("Available aws regions:") for item in list: count += 1 print("{}) {}".format(count, item)) return list # This function lists all available images from the providers. # Unlike GCE, the equivalent list_images() function requires NodeLocation as an argument so # we choose the first available one. # Without additional arguments like publisher, sku and version it takes rediculously # long to execute because there are over 3000 images available and the code that retrieves them has about 3 nested for loops. # That is why we use a dictionary to output the available images. The list does not get updated # but since the options include very old releases as well it is safe to assume these will also # be kept as available choices # Same goes for aws def list_provider_images(provider,images=None): driver = get_corresponding_driver(provider) print("Getting images from " +provider+"...") if provider == "azure": images = AZURE_images elif provider == "aws": images = AWS_images else: images = driver.list_images() # We filter out the images we do not care about images = list(filter(lambda x: 'windows' not in x.name.lower() and 'cos' not in x.name.lower() and 'arm64' not in x.name.lower() and 'byos' not in x.name.lower() and 'sap' not in x.name.lower(), images)) count = 0 print("Available "+provider+" images") if provider == "azure" or provider == "aws": for image in images: count += 1 print("{}) {}".format(count, image)) else: for image in images: count += 1 print("{}) {}\n\n{}\n".format(count, image.name, image.extra['description'])) return images def choose_from_list(listFromlistFunction,listName): if len(listFromlistFunction) == 0: print("No items") exit(0) if listName == "awsLocation": printFormat = "{}) {}\n\nRegion name: {}\nCountry: {}\n" printstring = "print(printFormat.format(count, item.name, item.availability_zone.region_name, item.country))" elif listName == "azureLocation" or listName == "gceLocation": printFormat = "{}) {}\n\nCountry: {}\n" printstring = "print(printFormat.format(count, item.name, item.country))" elif listName == "awsSize": printFormat = "{}) {}\n\nRam: {}\nDisk: {}\nBandwidth: {}\nPrice: {}\n" printstring = "print(printFormat.format(count, item.name, item.ram, item.disk, item.bandwidth, item.price))" elif listName == "gceSize": printFormat = "{}) {}\n\n{}\nPrice: {}\n" printstring = "print(printFormat.format(count, item.name, item.extra['description'], item.price))" elif listName == "azureSize": printFormat = "{}) {}\n\nRam: {}\nDisk: {}\nPrice: {}\n" printstring = "print(printFormat.format(count, item.name, item.ram, item.disk, item.price))" elif listName == "awsImage" or listName == "azureImage" or listName == "awsRegion": printFormat = "{}) {}" printstring = "print(printFormat.format(count, item))" elif listName == "gceImage": printFormat = "{}) {}\n\n{}\n" printstring = "print(printFormat.format(count, item.name, item.extra['description']))" elif listName == "node": printFormat = "{}) {}\n\nState: {}\nPublic IPs: {}\nPrivate IPs: {}\nDriver: {}\nSize: {}\nImage: {}\nCreation Date: {}\n" printstring = "print(printFormat.format(count, item.name, item.state, item.public_ips, item.private_ips, item.driver, item.size, item.image, item.created_at))" compiled_code = compile(printstring,"","exec") print("Choosing 0 will exit") choice = input("Choose the "+listName+" you want to use: ") while True: try: choice = int(choice) if choice > len(listFromlistFunction) or choice < 0: raise ValueError elif choice == 0: return elif choice == '': raise ValueError else: if listName == "azureImage": item = list(listFromlistFunction.values())[choice-1] elif listName == "awsImage": item = list(listFromlistFunction.values())[choice-1] print(list(listFromlistFunction)[choice-1]) elif listName == "awsRegion": item1 = list(listFromlistFunction)[choice-1] item2 = list(listFromlistFunction.values())[choice-1] return item1,item2 else: item = listFromlistFunction[int(choice) - 1] return item except ValueError: count = 0 for item in listFromlistFunction: count += 1 exec(compiled_code) print("Invalid choice") print("Choosing 0 will exit") choice = input("Choose the "+listName+" you want to use: ") def get_provider_location(provider): location = choose_from_list(list_provider_locations(provider),provider+"Location") return location def get_provider_size(provider): size = choose_from_list(list_provider_sizes(provider),provider+"Size") return size # This function asks the user which image he wants to use and returns the image name # If the user enters an invalid value, the program will ask again # If the user enters 0 the program will exit # For azure, after the user chooses one we take the value which is the image URN # and use it to get the actual AzureImage # For aws after we get the image we must select a region to get the ami because # amis are region specific. Then we get the actual image def get_provider_image(provider): image = choose_from_list(list_provider_images(provider),provider+"Image") if provider == "azure": driver = get_corresponding_driver(provider) azlocation = driver.list_locations()[0] if image is not None: image = driver.get_image(image,location=azlocation) elif provider == "aws": if image is not None: region = choose_from_list(listAWSregions(image),provider+"Region") if region is not None: dr = get_driver(Provider.EC2) correct_driver = dr(SECDEP_AWS_ACCESS_KEY, SECDEP_AWS_SECRET_KEY, region=region[0]) image = correct_driver.get_image(region[1]) else: image = None return image def blockPrint(): sys.stdout = open(os.devnull, 'w') def enablePrint(): sys.stdout = sys.__stdout__ def getAWSRegionFromAmi(ami): images = AWS_images.values() for image in images: if ami in image.values(): return list(image.keys())[list(image.values()).index(ami)] def create_node(provider, name=None, location=None, size=None, image=None, confirm=None): # Get public ssh key value with open(SECDEP_SSH_PUBLIC_KEY, 'r') as f: pubkey = f.read() if provider != "gce": auth = NodeAuthSSHKey(pubkey) # Check if name was given and if not prompt the user to give one if name is None: name = input("Enter the name of the node: ") assert name != "", "Name is empty" name = "provider_"+name # In the case of aws location has to be None because it is actually derived from the ami (image) if provider == "aws": location = None print("in aws you first have to choose an image before the location") else: # In other cases # Check if location was given and if not prompt the user to choose if location is None: location = get_provider_location(provider) assert location is not None, "Location is None" # If it was given, get the list of valid values to compare the imput with else: blockPrint() locations = list_provider_locations(provider) enablePrint() locationName = [] for loc in locations: # gce and aws have the location name under the name tag and azure under the id. if provider == "gce" or provider == "aws": locationName.append(loc.name) else: locationName.append(loc.id) # If it was not found prompt the user for selection if location not in locationName: print("Invalid Location") location = get_provider_location(provider) assert location is not None, "Location is None" else: location = locations[locationName.index(location)] # Check if size was given and if not prompt the user to choose if size is None: size = get_provider_size(provider) assert size is not None, "Size is None" # If it was given, get the list of valid values to compare the input with else: blockPrint() sizes = list_provider_sizes(provider) enablePrint() sizeName = [] for siz in sizes: sizeName.append(siz.name) if size not in sizeName: print("Invalid Size") size = get_provider_size(provider) assert size is not None, "Size is None" else: size = sizes[sizeName.index(size)] # Check if image was given and if not prompt the user to choose one if image is None: image = get_provider_image(provider) assert image is not None, "Image is None" else: blockPrint() # If provider was aws we must get the list of amis if provider != "aws": images = list_provider_images(provider) else: images = AWS_images.values() enablePrint() imageName = [] if provider != "aws": for img in images: if provider == "gce": imageName.append(img.name) elif provider == "azure": imageName.append(img.id) else: # In the case of aws this is how we get the list of amis for img in images: amis = img.values() for ami in amis: imageName.append(ami) if image not in imageName: print("Invalid Image") image = get_provider_image(provider) # If the image given was not in the list of valid values we must get the actual image and the region from the ami to get the correct driver if provider == "aws": ami = image.id region = getAWSRegionFromAmi(ami) dr = get_corresponding_driver(provider) assert dr is not None, "Driver is not set up correctly" image = get_driver(Provider.EC2)(SECDEP_AWS_ACCESS_KEY, SECDEP_AWS_SECRET_KEY,region=region).get_image(ami) assert image is not None, "Image is None" else: # If image was indeed in the list we choose it if provider != "aws": image = images[imageName.index(image)] else: # but in the case of aws we still need to get the region first region = getAWSRegionFromAmi(image) dr = get_corresponding_driver(provider) assert dr is not None, "Driver is not set up correctly" image = get_driver(Provider.EC2)(SECDEP_AWS_ACCESS_KEY, SECDEP_AWS_SECRET_KEY,region=region).get_image(image) # In the case of gce we need to give the sa_scopes and the ex_metadata parameters if provider == "gce": sa_scopes = [{"email": "default","scopes": ["cloud-platform"]}] ex_metadata = metadata = { "items": [{"key": "ssh-keys", "value": "secdep: %s" % (pubkey)}] } # For any other provider we just get the driver as defined in the get_corresponding_driver function if provider != "aws": driver = get_corresponding_driver(provider) else: # But for aws we need to get the one matching our derived region from the image selection dr = get_corresponding_driver(provider) assert dr is not None, "Driver is not set up correctly" driver = get_driver(Provider.EC2)(SECDEP_AWS_ACCESS_KEY, SECDEP_AWS_SECRET_KEY, region=region) # If the user did not input the -y or --yes flag then we output the current choices for a second though if confirm is False: print("\nName: %s\n" % (name)) # There is a differentiation between aws and the other providers in region and location if provider == "aws": print("\nLocation: %s\n" % (region)) else: print("\nLocation: %s\n" % (location)) print("\nSize: %s\n" % (size)) print("\nImage: %s\n" % (image)) print("Type yes if you want to confirm your choices") confirm = input("Continue? ") # Any input other than yes does not continue the node creation assert confirm == "yes", "User did not confirm" # In the case of gce the driver does not accept the auth parameter so we must create the node using the deploy node function if provider == "gce": # node = driver.create_node(name, size, image, location, ex_service_accounts=sa_scopes) # SCRIPT = """ # #!/usr/bin/env bash # useradd -G sudo -m secdep # echo "secdep:secdeppass" | chpasswd # mkdir -p /home/secdep/.ssh # """ # step = ScriptDeployment(SCRIPT) node = driver.create_node(name=name, image=image, size=size, location=location, ex_service_accounts=sa_scopes, ex_metadata=metadata) # node = driver.deploy_node(name=name, image=image, size=size, location=location, deploy=step, ex_service_accounts=sa_scopes, ex_metadata=metadata, ssh_key=SECDEP_SSH_PRIVATE_KEY) else: node = driver.create_node(name, size, image, location, auth=auth) else: # When the -y or --yes parameter is passed we go straight to the node creation if provider == "gce": # SCRIPT = '''#!/usr/bin/env bash # useradd -G sudo -m secdep # echo "secdep:secdeppass" | chpasswd # echo "%sudo ALL=(ALL:ALL) ALL" >> /etc/sudoers # mkdir -p /home/secdep/.ssh''' # step = ScriptDeployment(SCRIPT) node = driver.create_node(name=name, image=image, size=size, location=location, ex_service_accounts=sa_scopes, ex_metadata=metadata) # node = driver.deploy_node(name=name, image=image, size=size, location=location, deploy=step, ex_service_accounts=sa_scopes, ex_metadata=metadata, ssh_key=SECDEP_SSH_PRIVATE_KEY) else: node = driver.create_node(name, size, image, location, auth=auth) print(node.name + " created successfully") print("ip to connect to") print(node.public_ips) return node def list_all_nodes(): print("Getting all nodes...") nodes = [] if SECDEP_GCE_CLIENT_ID != "": driver = get_corresponding_driver("gce") gceNodes = driver.list_nodes() if len(gceNodes) > 0: for node in gceNodes: nodes.append(node) if SECDEP_AZURE_APP_ID != "": driver2 = get_corresponding_driver("azure") azureNodes = driver2.list_nodes() if len(azureNodes) > 0: for node in azureNodes: nodes.append(node) if SECDEP_AWS_ACCESS_KEY != "": awsLocations = ["ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2"] for region in awsLocations: driver3 = get_driver(Provider.EC2)(SECDEP_AWS_ACCESS_KEY, SECDEP_AWS_SECRET_KEY,region=region) # make it so it tries all drivers awsNodes = driver3.list_nodes() if len(awsNodes) > 0: for node in awsNodes: nodes.append(node) count = 0 if len(nodes) == 0: print("No nodes") exit(0) for node in nodes: count += 1 print("{}) {}\n\nState: {}\nPublic IPs: {}\nPrivate IPs: {}\nDriver: {}\nSize: {}\nImage: {}\nCreation Date: {}\n".format(count, node.name, node.state, node.public_ips, node.private_ips, node.driver, node.size, node.image, node.created_at)) return nodes def get_node(): node = choose_from_list(list_all_nodes(), "node") return node def delete_node(): node = choose_from_list(list_all_nodes(), "node") providerName = node.name.split("_")[0] if providerName == "gce": driver = get_corresponding_driver("gce") elif providerName == "azure": driver = get_corresponding_driver("azure") elif providerName == "aws": driver = get_corresponding_driver("aws") if(driver.destroy_node(node)): print("% node deleted successfully" % (providerName.upper())) else: print("% node could not be deleted" % (providerName.upper())) # def get_provider_features(provider): # driver = get_corresponding_driver(provider); # print(driver.features['create_node']) # # get_provider_features("azure") # If -p -q or -G is passed, provider must be passed as well if args.listimages or args.listsizes or args.listlocations: assert args.provider is not None, "Provider must be passed if listing images, sizes or locations" # If -v or --values is passed, call the update_env_file function if args.values: update_env_file() if args.listimages and args.provider: # If -p or --listimages is passed, call the list_provider_images function if args.print: print(get_provider_image(args.provider)) else: list_provider_images(args.provider) exit(0) if args.listsizes and args.provider: # If -q or --listsizes is passed, call the list_provider_sizes function if args.print: print(get_provider_size(args.provider)) else: list_provider_sizes(args.provider) exit(0) if args.listlocations and args.provider: # If -G or --listlocations is passed, call the list_provider_locations function if args.print: print(get_provider_location(args.provider)) else: list_provider_locations(args.provider) exit(0) if args.create: assert args.provider is not None, "Provider must be specified for node creation" # If -c or --create is passed, call the create_node function create_node(args.provider, args.name, args.region, args.size, args.image, args.yes) exit(0) if args.list: if args.print: print(get_node()) else: list_all_nodes() exit(0) if args.delete: delete_node() exit(0) # env = { # 'gce_key': gce_key, # 'gce_secret': gce_secret, # 'azure_key': azure_key, # 'azure_secret': azure_secret, # 'aws_key': aws_key, # 'aws_secret': aws_secret # } # with open('.env', 'w') as f: # json.dump(env, f)