- ### 개요: MSA 기반으로 각 기능을 수행하는 150 개의 람다, 레이어 배포
람다마다 설정이 달라서 별도로 처리해준 로직이 많아서 코드가 지저분해 보일 수 있습니다...
로그 그룹이 존재하면 기존 람다의 배포이고, 로그 그룹이 없다면 새로운 기능 배포인 케이스
## 소스코드
staging-lambda-function-deploy.yml
```
name: lambda function deploy to staging
on:
push:
branches:
- staging
jobs:
develop-lambbda-function-deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.DEVELOP_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.DEVELOP_AWS_SECRET_ACCESS_KEY }}
aws-region: ap-northeast-1
- name: aws layer deploy
run: |
bash deploy-layer.sh
env:
NODEJS_VERSION: nodejs14.x
AWS_ENV: stg
- name: aws lambda deploy
run: |
bash deploy-lambda.sh
env:
AWS_ACCOUNT_ID: ...
AWS_SUBNET_IDS: ...
AWS_SECURITY_GROUP_IDS: ...
AWS_ENV: stg
NODEJS_VERSION: nodejs14.x
```
deploy-layer.sh
```
#!/bin/bash
MAX_RETRY_CNT=5
LOG_DIR="./log/layer"
# To make a lot of requests at the same time
function request_aws_api () {
local function_name=${2}
local result
result=$(eval "${1} 2>&1")
if [[ $? -ne 0 ]]
then
output_error_logs "${1}" "${result}" "${function_name}"
if [[ $result =~ "Rate exceeded" ]]
then
result=$(retry_request_aws_api "${1}" "${function_name}")
else
exit 1
fi
fi
echo "result ${1}: ${result}" >&2
echo "result ${1}: ${result}" >> ${LOG_DIR}/${function_name}.log
echo $result
}
function rm_log_file () {
local function_name=${1}
if [ -e ${LOG_DIR}/${function_name}.log ]; then
rm ${LOG_DIR}/${function_name}.log
fi
}
function retry_request_aws_api () {
local function_name=${2}
for var in $(seq ${MAX_RETRY_CNT})
do
sleep 1
result=$(eval "${1} 2>&1");
if [[ $? -ne 0 ]]
then
output_error_logs "${1}" "${result}" "${function_name}"
else
echo $result
return
fi
done
exit 1
}
function output_error_logs () {
local cmd=${1};
local result=${2};
local function_name=${3};
echo "Request command: ${cmd}, Return value: ${result}" >> ${LOG_DIR}/${function_name}.log
}
if [ ! -d $LOG_DIR ]
then
mkdir -p $LOG_DIR
fi
mkdir deploy-layers
# layers zip
cd layers
if [ "${AWS_ENV}" = "prd" ]; then
cp ./cst-sdk/constants/CommonConstantProduction.js ./cst-sdk/constants/CommonConstant.js
cp ./cst-sdk/constants/ServersettingConstantProduction.js ./cst-sdk/constants/ServersettingConstant.js
fi
# package.jsonに記載されたパッケージをインストール
# その際、node_modulesディレクトリが作られる
npm install
mkdir nodejs
cp -R node_modules nodejs/
cp -R cst-sdk nodejs/node_modules
zip -r ../deploy-layers/cst-layers.zip nodejs -x /\*/node_modules/\*/node_modules/\* \*coverage\* \_\_test\_\_\* \*.DS_Store /
rm -R nodejs node_modules
cd ../
# layerをアップロード
cmd="aws lambda --no-cli-pager publish-layer-version --layer-name cst-sdk \
--description \"Layer for cpi setting tool.\" \
--zip-file fileb://deploy-layers/cst-layers.zip \
--compatible-runtimes ${NODEJS_VERSION} --region ap-northeast-1"
request_aws_api "${cmd}" "cst-layers"
echo "Process of deploying all layer is done!"
rm_log_file "cst-layers"
rm -rf ./deploy-layers
echo "============================================================================"
# error log output
if [ -d $LOG_DIR ]
then
error_flag=false
for err_log in $(ls ${LOG_DIR}); do
echo "============================================================================"
error_flag=true
cat ${LOG_DIR}/${err_log}
done
if [ "${error_flag}" = "true" ]
then
exit 1
fi
fi
```
deploy-lambda.sh
```
#!/bin/bash
MAX_RETRY_CNT=5
TIMEOUT_TEAMS_FUNCTION=cst-common-timeout-teams-notification-from-cloudwatch-log
LOG_DIR="./log/lambda"
# To make a lot of requests at the same time
function request_aws_api () {
local function_name=${2}
local result
result=$(eval "${1} 2>&1")
if [[ $? -ne 0 ]]
then
output_error_logs "${1}" "${result}" "${function_name}"
if [[ $result =~ "Rate exceeded" ]]
then
result=$(retry_request_aws_api "${1}" "${function_name}")
elif [[ ( $result =~ "ResourceNotFoundException" ) && ( ${1} =~ "get-function-configuration" ) ]]
then
result=false
else
exit 1
fi
fi
echo "result ${1}: ${result}" >&2
echo "result ${1}: ${result}" >> ${LOG_DIR}/${function_name}.log
echo $result
}
function rm_log_file () {
local function_name=${1}
if [ -e ${LOG_DIR}/${function_name}.log ]; then
rm ${LOG_DIR}/${function_name}.log
fi
}
function retry_request_aws_api () {
local function_name=${2}
for var in $(seq ${MAX_RETRY_CNT})
do
sleep 1
result=$(eval "${1} 2>&1");
if [[ $? -ne 0 ]]
then
output_error_logs "${1}" "${result}" "${function_name}"
if [[ ( $result =~ "ResourceNotFoundException" ) && ( ${1} =~ "get-function-configuration" ) ]]
then
echo false
return
fi
else
echo $result
return
fi
done
exit 1
}
function output_error_logs () {
local cmd=${1};
local result=${2};
local function_name=${3};
echo "Request command: ${cmd}, Return value: ${result}" >> ${LOG_DIR}/${function_name}.log
}
function log_groups_exists () {
local function_name=${1}
local logs_group_name=/aws/lambda/${function_name}
local cmd="aws logs describe-log-groups --no-cli-pager --log-group-name-prefix ${logs_group_name} \
--query 'logGroups[].logGroupName'"
local result=$(request_aws_api "${cmd}" "log_groups-${function_name}")
local log_groups_exists_flag=`echo ${result} | awk -F'\"' '{print \$2}' | grep -e "^${logs_group_name}$" || true`
if [ -z "$log_groups_exists_flag" ]
then
echo false
else
echo true
fi
}
function create_log_groups () {
local logs_group_name=/aws/lambda/${1}
local log_groups_exists_flag=$(log_groups_exists "${1}")
if [ "$log_groups_exists_flag" = "false" ]
then
local cmd
cmd="aws logs create-log-group --log-group-name ${logs_group_name}"
request_aws_api "${cmd}" "log_groups-${1}"
fi
rm_log_file "log_groups-${1}"
}
function get_timeout () {
local directory=${1};
local timeout=120
if [ ${directory%.zip} = cst-sslsettings-approval-check-from-cloudwatch-rule ]; then
timeout=900
fi
if [ ${directory%.zip} = cst-sslsettings-resend-check-from-cloudwatch-rule ]; then
timeout=300
fi
if [ ${directory%.zip} = cst-sslsetting-calling-copy-from-sns ]; then
timeout=180
fi
if [ ${directory%.zip} = cst-sslsetting-calling-put-pem-from-sns ]; then
timeout=240
fi
if [ ${directory%.zip} = cst-serversettings-auto-stop-from-cloudwatch-rule ]; then
timeout=300
fi
if [ ${directory%.zip} = cst-serversettings-auto-revival-from-cloudwatch-rule ]; then
timeout=300
fi
if [ ${directory%.zip} = cst-serversettings-auto-delete-from-cloudwatch-rule ]; then
timeout=300
fi
if [ ${directory%.zip} = cst-serversettings-auto-add-from-cloudwatch-rule ]; then
timeout=300
fi
if [ ${directory%.zip} = cst-sslsettings-auto-cpi-ssl-from-cloudwatch-rule ]; then
timeout=300
fi
if [ ${directory%.zip} = cst-sslsettings-auto-other-ssl-from-cloudwatch-rule ]; then
timeout=300
fi
if [ ${directory%.zip} = cst-opesetting-calling-ssl-install-from-sns ]; then
timeout=300
fi
if [ ${directory%.zip} = cst-opesetting-callback-ssl-install-from-sns ]; then
timeout=300
fi
if [ ${directory%.zip} = cst-serversetting-calling-virtualmod-step2-from-sns ]; then
timeout=900
fi
echo $timeout
}
function back_ground_process () {
local directory=${1};
local timeout=$(get_timeout "${directory}")
local cmd=""
echo deploy-functions/${directory};
local function_name=${directory%.zip}
local function_exists_flag=$(function_exists ${function_name});
if [ "$function_exists_flag" = "false" ]
then
cmd="aws lambda create-function --no-cli-pager \
--function-name ${function_name} \
--runtime ${NODEJS_VERSION} \
--zip-file fileb://deploy-functions/${directory} \
--handler index.handler \
--role arn:aws:iam::${AWS_ACCOUNT_ID}:role/cpi-setting-tool-lambda-role-access-dynamodb-sns-vpc-ses \
--environment \"Variables={ENV=${AWS_ENV}}\"";
request_aws_api "${cmd}" "${function_name}"
fi
cmd="aws lambda update-function-code --no-cli-pager --function-name ${function_name} \
--zip-file fileb://deploy-functions/${directory} \
--publish ";
request_aws_api "${cmd}" "${function_name}"
echo "waiting for lambda to update"
cmd="aws lambda wait function-updated --no-cli-pager --function-name ${function_name}";
request_aws_api "${cmd}" "${function_name}"
cmd="aws lambda update-function-configuration --no-cli-pager \
--function-name ${function_name} \
--role arn:aws:iam::${AWS_ACCOUNT_ID}:role/cpi-setting-tool-lambda-role-access-dynamodb-sns-vpc-ses \
--runtime ${NODEJS_VERSION} \
--vpc-config SubnetIds=${AWS_SUBNET_IDS},SecurityGroupIds=${AWS_SECURITY_GROUP_IDS} \
--timeout $timeout \
--layers arn:aws:lambda:ap-northeast-1:${AWS_ACCOUNT_ID}:layer:cst-sdk:${layer_version} \
--environment \"Variables={ENV=${AWS_ENV}}\"";
request_aws_api "${cmd}" "${function_name}"
cmd="aws lambda put-function-event-invoke-config --no-cli-pager \
--function-name ${function_name} \
--maximum-retry-attempts 0";
request_aws_api "${cmd}" "${function_name}"
if [ "${directory%.zip}" = "${TIMEOUT_TEAMS_FUNCTION}" ]
then
rm_log_file "${TIMEOUT_TEAMS_FUNCTION}"
return
fi
logs_group_name=/aws/lambda/${function_name}
create_log_groups "${function_name}"
cmd="aws logs put-subscription-filter --no-cli-pager \
--log-group-name ${logs_group_name} \
--filter-name taskTimedOutAfter \
--filter-pattern \"Task timed out after\" \
--destination-arn arn:aws:lambda:ap-northeast-1:${AWS_ACCOUNT_ID}:function:${TIMEOUT_TEAMS_FUNCTION}";
request_aws_api "${cmd}" "${function_name}"
rm_log_file "${function_name}"
echo "${1} done!"
}
function function_exists () {
local function_name=${1}
local cmd="aws lambda --no-cli-pager get-function-configuration \
--function-name ${function_name}"
local result=$(request_aws_api "${cmd}" "${function_name}")
if [ "$result" = "false" ]
then
echo false
else
echo true
fi
}
if [ ! -d $LOG_DIR ]
then
mkdir -p $LOG_DIR
fi
mkdir deploy-functions
# function zip
for directory in $(ls ./functions); do
cd functions/${directory}
zip -r ../../deploy-functions/${directory}.zip . -x /package.json package-lock.json node_modules\* \*coverage\* \_\_test\_\_\* \*.DS_Store /
cd ../../
done
cmd="aws lambda --no-cli-pager list-layer-versions --layer-name cst-sdk --region ap-northeast-1 --query 'LayerVersions[0].Version'"
layer_version=$(request_aws_api "${cmd}" "layer_version")
echo layer_version=$layer_version
rm_log_file "layer_version"
# {TIMEOUT_TEAMS_FUNCTION}は他の lambda と設定内容が違うので個別に作成
function_exists_flag=$(function_exists ${TIMEOUT_TEAMS_FUNCTION});
if [ "$function_exists_flag" = "false" ]
then
cmd="aws lambda create-function --no-cli-pager \
--function-name ${TIMEOUT_TEAMS_FUNCTION} \
--runtime ${NODEJS_VERSION} \
--zip-file fileb://deploy-functions/${TIMEOUT_TEAMS_FUNCTION}.zip \
--handler index.handler \
--role arn:aws:iam::${AWS_ACCOUNT_ID}:role/cpi-setting-tool-lambda-role-access-dynamodb-sns-vpc-ses";
request_aws_api "${cmd}" "${TIMEOUT_TEAMS_FUNCTION}"
fi
cmd="aws lambda remove-permission --no-cli-pager \
--function-name ${TIMEOUT_TEAMS_FUNCTION} \
--statement-id CloudWatch-Logs-filter-taskTimedOutAfter || true";
request_aws_api "${cmd}" "${TIMEOUT_TEAMS_FUNCTION}"
cmd="aws lambda add-permission --no-cli-pager \
--function-name ${TIMEOUT_TEAMS_FUNCTION} \
--action lambda:InvokeFunction \
--statement-id CloudWatch-Logs-filter-taskTimedOutAfter \
--principal logs.ap-northeast-1.amazonaws.com \
--source-arn \"arn:aws:logs:ap-northeast-1:${AWS_ACCOUNT_ID}:log-group:/aws/lambda/cst-*:*\"";
request_aws_api "${cmd}" "${TIMEOUT_TEAMS_FUNCTION}"
# zip and upload each function
for directory in $(ls ./deploy-functions); do
back_ground_process ${directory} &
done
# wait until all child processes are done
wait
echo "All background processes are done!"
rm -rf ./deploy-functions
echo "============================================================================"
# error log output
if [ -d $LOG_DIR ]
then
error_flag=false
for err_log in $(ls ${LOG_DIR}); do
echo "============================================================================"
error_flag=true
cat ${LOG_DIR}/${err_log}
done
if [ "$error_flag" = "true" ]
then
exit 1
fi
fi
```
