Terraform das variáveis do programa e políticas IAM.
Variáveis
variables.tf
variable "repository_branch" {
description = "Repository branch to use in the pipeline"
default = "dev"
}
variable "env" {
description = "Deployment environment"
default = "dev"
}
variable "region" {
description = "AWS region"
default = "us-east-1"
}
variable "build_repository_name" {
description = "Build repository name"
# Aqui fica o nome do repositório para o código do modelo
default = "model-build-pipeline"
}
variable "deploy_repository_name" {
description = "Deploy repository name"
default = "model-deploy-pipeline"
}
variable "artifacts_bucket_name" {
description = "S3 Bucket for storing artifacts"
default = "sagemaker-artifacts-pipeline-18345"
}
variable "project_name" {
description = "Project name"
default = "aws-sagemaker-terraform-cicd"
}
variable "project_id" {
description = "Project ID"
default = "10072022"
}Nas variáveis, escolhemos os nomes para o S3, do projeto, dos repositórios de build e deploy, assim como a branch que será usada dos repositórios na pipeline. O id do projeto serve para identificar unicamente ele, e é exigido pelo SageMaker.
Armazenamento no S3
resource "aws_s3_bucket" "artifacts_bucket" {
bucket = var.artifacts_bucket_name
force_destroy = true
}Neste arquivo temos apenas a criação do bucket S3 que será utilizado para guardar tudo da pipeline, usamos o nome definido nas variáveis e o force_destroy para poder destruir ele junto com o terraform destroy sem ter que esvaziar primeiro.
Roles IAM
iam_roles.tf
resource "aws_iam_role" "tf_mlops_role" {
assume_role_policy = jsonencode(
{
Statement = [
{
Action = "sts:AssumeRole"
Effect = "Allow"
Principal = {
Service = [
"firehose.amazonaws.com",
"glue.amazonaws.com",
"apigateway.amazonaws.com",
"lambda.amazonaws.com",
"events.amazonaws.com",
"states.amazonaws.com",
"sagemaker.amazonaws.com",
"cloudformation.amazonaws.com",
"codebuild.amazonaws.com",
"codepipeline.amazonaws.com",
"codecommit.amazonaws.com",
]
}
},
]
Version = "2012-10-17"
}
)
force_detach_policies = false
max_session_duration = 3600
name = "tf-mlops-role-${var.env}-1007"
path = "/service-role/"
tags = {}
}
resource "aws_iam_policy" "tf_mlops_policy" {
description = "Policy used in trust relationship with CodeBuild (${var.env})"
name = "tf-mlops-policy-${var.env}-1007"
path = "/service-role/"
policy = jsonencode(
{
Statement = [
{
"Effect" = "Allow",
"Action" = [
"codecommit:GitPull",
"codecommit:UploadArchive",
"codecommit:GetBranch",
"codecommit:GetCommit",
"codecommit:GetUploadArchiveStatus",
"codecommit:CancelUploadArchive",
],
"Resource" = [
"arn:aws:codecommit:*:*:*",
]
},
{
Action : [
"iam:PassRole"
],
Resource : "*",
Effect : "Allow"
},
{
"Effect" = "Allow",
"Action" = [
"s3:*"
],
"Resource" = [
"arn:aws:s3:::*",
"arn:aws:s3:::*"
]
},
{
"Action" : [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Effect" : "Allow",
"Resource" : "arn:aws:logs:*"
},
{
"Action": [
"codepipeline:StartPipelineExecution"
],
"Resource": "arn:aws:codepipeline:*:*:*",
"Effect": "Allow"
},
{
"Action": [
"events:DeleteRule",
"events:DescribeRule",
"events:PutRule",
"events:PutTargets",
"events:RemoveTargets"
],
"Resource": [
"arn:aws:events:*:*:rule/*"
],
"Effect": "Allow"
},
{
"Effect": "Allow",
"Action": [
"sagemaker:*"
],
"NotResource": [
"arn:aws:sagemaker:*:*:domain/*",
"arn:aws:sagemaker:*:*:user-profile/*",
"arn:aws:sagemaker:*:*:app/*",
"arn:aws:sagemaker:*:*:flow-definition/*"
]
},
{
"Effect": "Allow",
"Action": [
"sagemaker:CreatePresignedDomainUrl",
"sagemaker:DescribeDomain",
"sagemaker:ListDomains",
"sagemaker:DescribeUserProfile",
"sagemaker:ListUserProfiles",
"sagemaker:*App",
"sagemaker:ListApps"
],
"Resource": "*"
},
{
"Effect": "Allow",
"Action": "sagemaker:*",
"Resource": [
"arn:aws:sagemaker:*:*:flow-definition/*"
],
"Condition": {
"StringEqualsIfExists": {
"sagemaker:WorkteamType": [
"private-crowd",
"vendor-crowd"
]
}
}
},
{
"Action": [
"cloudformation:CreateChangeSet",
"cloudformation:CreateStack",
"cloudformation:DescribeChangeSet",
"cloudformation:DeleteChangeSet",
"cloudformation:DeleteStack",
"cloudformation:DescribeStacks",
"cloudformation:ExecuteChangeSet",
"cloudformation:SetStackPolicy",
"cloudformation:UpdateStack"
],
"Resource": "arn:aws:cloudformation:*:*:stack/*",
"Effect": "Allow"
},
{
"Action": [
"codebuild:BatchGetBuilds",
"codebuild:StartBuild"
],
"Resource": [
"arn:aws:codebuild:*:*:project/*",
"arn:aws:codebuild:*:*:build/*"
],
"Effect": "Allow"
},
{
"Action": [
"states:DescribeExecution",
"states:GetExecutionHistory",
"states:StartExecution",
"states:StopExecution",
"states:UpdateStateMachine"
],
"Resource": [
"arn:aws:states:*:*:statemachine:*sagemaker*",
"arn:aws:states:*:*:execution:*sagemaker*:*"
],
"Effect": "Allow"
},
{
"Effect": "Allow",
"Action": [
"secretsmanager:DescribeSecret",
"secretsmanager:GetSecretValue",
"secretsmanager:CreateSecret"
],
"Resource": [
"arn:aws:secretsmanager:*:*:secret:AmazonSageMaker-*"
]
},
{
"Effect": "Allow",
"Action": [
"secretsmanager:DescribeSecret",
"secretsmanager:GetSecretValue"
],
"Resource": "*",
"Condition": {
"StringEquals": {
"secretsmanager:ResourceTag/SageMaker": "true"
}
}
},
{
"Effect": "Allow",
"Action": [
"s3:CreateBucket",
"s3:GetBucketLocation",
"s3:ListBucket",
"s3:ListAllMyBuckets",
"s3:GetBucketCors",
"s3:PutBucketCors"
],
"Resource": "*"
},
{
"Effect": "Allow",
"Action": [
"s3:GetBucketAcl",
"s3:PutObjectAcl"
],
"Resource": [
"arn:aws:s3:::*SageMaker*",
"arn:aws:s3:::*Sagemaker*",
"arn:aws:s3:::*sagemaker*"
]
},
{
"Action": [
"ecr:BatchCheckLayerAvailability",
"ecr:BatchGetImage",
"ecr:Describe*",
"ecr:GetAuthorizationToken",
"ecr:GetDownloadUrlForLayer"
],
"Resource": "*",
"Effect": "Allow"
},
{
"Effect": "Allow",
"Action": [
"ecr:BatchDeleteImage",
"ecr:CompleteLayerUpload",
"ecr:CreateRepository",
"ecr:DeleteRepository",
"ecr:InitiateLayerUpload",
"ecr:PutImage",
"ecr:UploadLayerPart"
],
"Resource": [
"arn:aws:ecr:*:*:repository/sagemaker-*"
]
},
{
"Action": "iam:CreateServiceLinkedRole",
"Effect": "Allow",
"Resource": "arn:aws:iam::*:role/aws-service-role/sagemaker.application-autoscaling.amazonaws.com/AWSServiceRoleForApplicationAutoScaling_SageMakerEndpoint",
"Condition": {
"StringLike": {
"iam:AWSServiceName": "sagemaker.application-autoscaling.amazonaws.com"
}
}
},
{
"Effect": "Allow",
"Action": "iam:CreateServiceLinkedRole",
"Resource": "*",
"Condition": {
"StringEquals": {
"iam:AWSServiceName": "robomaker.amazonaws.com"
}
}
},
{
"Effect": "Allow",
"Action": [
"sns:Subscribe",
"sns:CreateTopic"
],
"Resource": [
"arn:aws:sns:*:*:*SageMaker*",
"arn:aws:sns:*:*:*Sagemaker*",
"arn:aws:sns:*:*:*sagemaker*"
]
},
{
"Effect": "Allow",
"Action": [
"iam:PassRole"
],
"Resource": "arn:aws:iam::*:role/*AmazonSageMaker*",
"Condition": {
"StringEquals": {
"iam:PassedToService": [
"glue.amazonaws.com",
"robomaker.amazonaws.com",
"states.amazonaws.com"
]
}
}
},
{
"Effect": "Allow",
"Action": [
"iam:PassRole"
],
"Resource": "arn:aws:iam::*:role/*",
"Condition": {
"StringEquals": {
"iam:PassedToService": "sagemaker.amazonaws.com"
}
}
}
]
Version = "2012-10-17"
}
)
}
resource "aws_iam_role_policy_attachment" "tf_mlops_policy_attachment" {
role = aws_iam_role.tf_mlops_role.name
policy_arn = aws_iam_policy.tf_mlops_policy.arn
}Neste arquivo fazemos a definição da service role que será utilizada na pipeline, por isso é importante que todos os serviços que serão utilizados durante qualquer fase do projeto sejam incluídos devidamente, pois se faltar a permissão para qualquer coisa, teremos erros durante a execução da pipeline. Muitos desses serviços são utilizados durante a pipeline do SageMaker e/ou na etapa de deploy do modelo pelo cloudformation.