1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182AWSTemplateFormatVersion: 2010-09-09
Description: >
Creates a new AWS VPC. It then deploys an ECS task definition, required IAM roles, and ECS service
running a Prefect Streaming Service in a subnet created within that VPC
Parameters:
project:
Type: String
Description: Project name
Default: realtime-poc
cluster:
Type: String
Description: ECS Cluster name
Default: prefect-streaming
aws_account_id:
Type: String
Description: AWS Account ID
Default: 338306982838
region:
Type: String
Description: AWS region name
Default: us-east-1
cpu:
Type: String
Description: Allow Dynamic CPU configuration
Default: 512
AllowedValues: [256, 512, 1024, 2048, 4096]
memory:
Type: String
Description: Allow Increasing Memory - from 8192 on requires 4096 CPU and increases in 1024 increments
Default: 1024
AllowedValues: [512, 1024, 2048, 4096, 5120, 6144, 7168, 8192 , 9216, 10240]
image:
Type: String
Description: Docker image for the service
Default: prefecthq/prefect:2.0b8-python3.9
Resources:
PrefectFargateCluster:
Type: AWS::ECS::Cluster
Properties:
ClusterName: !Ref cluster
PrefectLogGroup:
Type: AWS::Logs::LogGroup
Properties:
LogGroupName: !Ref project
RetentionInDays: 7
PrefectVPC:
Type: AWS::EC2::VPC
Properties:
CidrBlock: 10.0.0.0/16
EnableDnsSupport: true
EnableDnsHostnames: true
InternetGateway:
Type: AWS::EC2::InternetGateway
InternetGatewayAttachment:
Type: AWS::EC2::VPCGatewayAttachment
Properties:
VpcId: !Ref PrefectVPC
InternetGatewayId: !Ref InternetGateway
PublicRouteTable:
Type: AWS::EC2::RouteTable
Properties:
VpcId: !Ref PrefectVPC
RouteToGateway:
Type: AWS::EC2::Route
DependsOn: InternetGatewayAttachment
Properties:
RouteTableId: !Ref PublicRouteTable
DestinationCidrBlock: 0.0.0.0/0
GatewayId: !Ref InternetGateway
PrefectECSServiceSubnet:
Type: AWS::EC2::Subnet
Properties:
VpcId: !Ref PrefectVPC
CidrBlock: 10.0.0.0/16
AvailabilityZone:
Fn::Select:
- 0
- Fn::GetAZs: { Ref: 'AWS::Region' }
MapPublicIpOnLaunch: true
SubnetRouteTableAssociation:
Type: AWS::EC2::SubnetRouteTableAssociation
Properties:
SubnetId: !Ref PrefectECSServiceSubnet
RouteTableId: !Ref PublicRouteTable
ExecutionRole:
Type: AWS::IAM::Role
Properties:
RoleName: !Sub "${project}_ecs_execution_role"
AssumeRolePolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Principal:
Service: ecs-tasks.amazonaws.com
Action: sts:AssumeRole
Policies:
- PolicyName: AllowRetrievingSecretsFromParameterStore
PolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Action:
- ssm:GetParameters
Resource: "*"
ManagedPolicyArns:
- arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
TaskRole:
Type: AWS::IAM::Role
Properties:
RoleName: !Sub "${project}_ecs_task_role"
AssumeRolePolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Principal:
Service: ecs-tasks.amazonaws.com
Action: sts:AssumeRole
Policies:
- PolicyName: PrefectAthenaS3DataLake
PolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Action:
- athena:*
- glue:*
- s3:*
Resource: "*"
PrefectTaskDefinition:
Type: AWS::ECS::TaskDefinition
Properties:
Family: !Ref project
Cpu: !Ref cpu
Memory: !Ref memory
NetworkMode: awsvpc
ExecutionRoleArn: !Ref ExecutionRole
TaskRoleArn: !Ref TaskRole
ContainerDefinitions:
- Name: !Ref project
Image: !Ref image
LogConfiguration:
LogDriver: awslogs
Options:
awslogs-region: !Ref AWS::Region
awslogs-group: !Ref PrefectLogGroup
awslogs-stream-prefix: !Ref project
Secrets:
- Name: PREFECT_API_URL
ValueFrom: !Sub "arn:aws:ssm:${region}:${aws_account_id}:parameter/PREFECT_API_URL"
- Name: PREFECT_API_KEY
ValueFrom: !Sub "arn:aws:ssm:${region}:${aws_account_id}:parameter/PREFECT_API_KEY"
# - Name: SLACK_WEBHOOK_URL
# ValueFrom: !Sub "arn:aws:ssm:${region}:${aws_account_id}:parameter/SLACK_WEBHOOK_URL"
RequiresCompatibilities:
- FARGATE
PrefectECSService:
Type: AWS::ECS::Service
DependsOn:
- SubnetRouteTableAssociation
- RouteToGateway
- PrefectFargateCluster
Properties:
ServiceName: !Ref project
Cluster: !Ref PrefectFargateCluster
TaskDefinition: !Ref PrefectTaskDefinition
DesiredCount: 1
LaunchType: FARGATE
NetworkConfiguration:
AwsvpcConfiguration:
AssignPublicIp: ENABLED
Subnets:
- !Ref PrefectECSServiceSubnet