From 23d89ecb01a8ab97f2b7d8b08662902425d8701d Mon Sep 17 00:00:00 2001 From: Erin Weisbart <54687786+ErinWeisbart@users.noreply.github.com> Date: Fri, 24 Mar 2023 15:43:24 -0700 Subject: [PATCH 1/2] launches instances using ec2.create_fleet but not finished --- fleet.py | 5 ++++ run.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 fleet.py diff --git a/fleet.py b/fleet.py new file mode 100644 index 0000000..4aa4a89 --- /dev/null +++ b/fleet.py @@ -0,0 +1,5 @@ +ImageId = "ami-fad25980" +KeyName = "your_key_file_name" +SnapshotId = "snap-04007a196c0f3f398" +SubnetId = "subnet-WWWWWWWW" +SecurityGroup= "sg-ZZZZZZZZZ" diff --git a/run.py b/run.py index be29383..8d6f6d7 100644 --- a/run.py +++ b/run.py @@ -13,6 +13,7 @@ CLEAN_DASHBOARD = 'False' from config import * +from fleet import * WAIT_TIME = 60 MONITOR_TIME = 60 @@ -557,26 +558,84 @@ def startCluster(): print('Use: run.py startCluster configFile') sys.exit() - thistime = datetime.datetime.now().replace(microsecond=0) - #Step 1: set up the configuration files + #Step 1: Configure the Spot Fleet Request + ec2client=boto3.client('ec2') s3client = boto3.client('s3') + iam=boto3.client('iam') + ecsConfigFile=generateECSconfig(ECS_CLUSTER,APP_NAME,AWS_BUCKET,s3client) spotfleetConfig=loadConfig(sys.argv[2]) - spotfleetConfig['ValidFrom']=thistime - spotfleetConfig['ValidUntil']=(thistime+datetime.timedelta(days=365)).replace(microsecond=0) - spotfleetConfig['TargetCapacity']= CLUSTER_MACHINES - spotfleetConfig['SpotPrice'] = '%.2f' %MACHINE_PRICE + + # Still need to figure out how these fit in + """ DOCKER_BASE_SIZE = int(round(float(EBS_VOL_SIZE)/int(TASKS_PER_MACHINE))) - 2 userData=generateUserData(ecsConfigFile,DOCKER_BASE_SIZE) for LaunchSpecification in range(0,len(spotfleetConfig['LaunchSpecifications'])): spotfleetConfig['LaunchSpecifications'][LaunchSpecification]["UserData"]=userData spotfleetConfig['LaunchSpecifications'][LaunchSpecification]['BlockDeviceMappings'][1]['Ebs']["VolumeSize"]= EBS_VOL_SIZE - spotfleetConfig['LaunchSpecifications'][LaunchSpecification]['InstanceType'] = MACHINE_TYPE[LaunchSpecification] + """ + ecsInstanceArn = iam.get_role(RoleName='ecsInstanceRole')['Role']['Arn'] + fleetRoleArn = iam.get_role(RoleName='aws-ec2-spot-fleet-tagging-role')['Role']['Arn'] - # Step 2: make the spot fleet request - ec2client=boto3.client('ec2') - requestInfo = ec2client.request_spot_fleet(SpotFleetRequestConfig=spotfleetConfig) + LaunchTemplateData={"ImageId": ImageId, + "KeyName": KeyName, + "IamInstanceProfile": {"Arn": ecsInstanceArn}, + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "DeleteOnTermination": True, + "VolumeType": "gp2", + "VolumeSize": 8, + "SnapshotId": SnapshotId + } + }, + { + "DeviceName": "/dev/xvdcz", + "Ebs": { + "DeleteOnTermination": True, + "VolumeType": "gp2" + } + } + ], + "NetworkInterfaces": [ + { + "DeviceIndex": 0, + "SubnetId": SubnetId, + "DeleteOnTermination": True, + "AssociatePublicIpAddress": True, + "Groups": [SecurityGroup] + } + ] + } + + TemplateName=f'{APP_NAME}_LaunchTemplate' + try: + launch_template = ec2client.describe_launch_templates(LaunchTemplateNames=[TemplateName])['LaunchTemplates'][0] + except ec2client.exceptions.ClientError: + launch_template = ec2client.create_launch_template(LaunchTemplateName=TemplateName,LaunchTemplateData=LaunchTemplateData)['LaunchTemplate'] + + SpotOptions = {"AllocationStrategy": "lowestPrice"} + LaunchTemplateConfigs=[{'LaunchTemplateSpecification':{'LaunchTemplateId':launch_template['LaunchTemplateId'], + 'Version':'$Latest'}, + 'Overrides':[{'InstanceType':MACHINE_TYPE, + 'MaxPrice':'%.2f' %MACHINE_PRICE, + 'SubnetId':SubnetId}]}] + TargetCapacitySpecification={'TotalTargetCapacity':CLUSTER_MACHINES, + 'OnDemandTargetCapacity':0, + 'DefaultTargetCapacityType':'spot'} + + # Step 2: Make the spot fleet request + thistime = datetime.datetime.now().replace(microsecond=0) + requestInfo = ec2client.create_fleet(DryRun=False, + SpotOptions=SpotOptions, + LaunchTemplateConfigs=LaunchTemplateConfigs, + TargetCapacitySpecification=TargetCapacitySpecification, + TerminateInstancesWithExpiration=True, + Type="maintain", + ValidFrom=thistime, + ValidUntil=(thistime+datetime.timedelta(days=365)).replace(microsecond=0),) print('Request in process. Wait until your machines are available in the cluster.') print('SpotFleetRequestId',requestInfo['SpotFleetRequestId']) From c07f3da13309ef160fdb221a48b7b54a74df3268 Mon Sep 17 00:00:00 2001 From: Erin Weisbart <54687786+ErinWeisbart@users.noreply.github.com> Date: Fri, 24 Mar 2023 22:00:49 -0700 Subject: [PATCH 2/2] progress on spot fleet request --- run.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/run.py b/run.py index 8d6f6d7..5cc369f 100644 --- a/run.py +++ b/run.py @@ -564,22 +564,13 @@ def startCluster(): iam=boto3.client('iam') ecsConfigFile=generateECSconfig(ECS_CLUSTER,APP_NAME,AWS_BUCKET,s3client) - spotfleetConfig=loadConfig(sys.argv[2]) - - # Still need to figure out how these fit in - """ DOCKER_BASE_SIZE = int(round(float(EBS_VOL_SIZE)/int(TASKS_PER_MACHINE))) - 2 userData=generateUserData(ecsConfigFile,DOCKER_BASE_SIZE) - for LaunchSpecification in range(0,len(spotfleetConfig['LaunchSpecifications'])): - spotfleetConfig['LaunchSpecifications'][LaunchSpecification]["UserData"]=userData - spotfleetConfig['LaunchSpecifications'][LaunchSpecification]['BlockDeviceMappings'][1]['Ebs']["VolumeSize"]= EBS_VOL_SIZE - """ - ecsInstanceArn = iam.get_role(RoleName='ecsInstanceRole')['Role']['Arn'] + ecsInstanceArn = iam.get_instance_profile(InstanceProfileName='ecsInstanceRole')['InstanceProfile']['Arn'] fleetRoleArn = iam.get_role(RoleName='aws-ec2-spot-fleet-tagging-role')['Role']['Arn'] - LaunchTemplateData={"ImageId": ImageId, - "KeyName": KeyName, + LaunchTemplateData={ "IamInstanceProfile": {"Arn": ecsInstanceArn}, "BlockDeviceMappings": [ { @@ -587,7 +578,7 @@ def startCluster(): "Ebs": { "DeleteOnTermination": True, "VolumeType": "gp2", - "VolumeSize": 8, + "VolumeSize": EBS_VOL_SIZE, "SnapshotId": SnapshotId } }, @@ -607,19 +598,23 @@ def startCluster(): "AssociatePublicIpAddress": True, "Groups": [SecurityGroup] } - ] + ], + "ImageId": ImageId, + "InstanceType": MACHINE_TYPE[0], + "KeyName": KeyName, + "UserData": userData } TemplateName=f'{APP_NAME}_LaunchTemplate' try: - launch_template = ec2client.describe_launch_templates(LaunchTemplateNames=[TemplateName])['LaunchTemplates'][0] + launch_template = ec2client.create_launch_template_version(LaunchTemplateName=TemplateName,LaunchTemplateData=LaunchTemplateData)['LaunchTemplateVersion'] except ec2client.exceptions.ClientError: launch_template = ec2client.create_launch_template(LaunchTemplateName=TemplateName,LaunchTemplateData=LaunchTemplateData)['LaunchTemplate'] SpotOptions = {"AllocationStrategy": "lowestPrice"} LaunchTemplateConfigs=[{'LaunchTemplateSpecification':{'LaunchTemplateId':launch_template['LaunchTemplateId'], 'Version':'$Latest'}, - 'Overrides':[{'InstanceType':MACHINE_TYPE, + 'Overrides':[{ 'MaxPrice':'%.2f' %MACHINE_PRICE, 'SubnetId':SubnetId}]}] TargetCapacitySpecification={'TotalTargetCapacity':CLUSTER_MACHINES, @@ -628,7 +623,7 @@ def startCluster(): # Step 2: Make the spot fleet request thistime = datetime.datetime.now().replace(microsecond=0) - requestInfo = ec2client.create_fleet(DryRun=False, + requestInfo = ec2client.create_fleet( SpotOptions=SpotOptions, LaunchTemplateConfigs=LaunchTemplateConfigs, TargetCapacitySpecification=TargetCapacitySpecification,