Actions Runner Controller 구성

helm repo add jetstack https://charts.jetstack.io
helm repo update

helm install \
  cert-manager jetstack/cert-manager \
  --namespace cert-manager \
  --create-namespace \
  --version v1.12.0 \
  --set installCRDs=true \
  --wait

$ kubectl get pod -n cert-manager
NAME                                      READY   STATUS    RESTARTS   AGE
cert-manager-74654c4948-b5n2c             1/1     Running   0          19s
cert-manager-cainjector-77644bff8-49rpl   1/1     Running   0          19s
cert-manager-webhook-54d7657dbb-5hshh     1/1     Running   0          19s

$ kubectl api-resources --categories cert-manager
NAME                  SHORTNAMES   APIVERSION                NAMESPACED   KIND
challenges                         acme.cert-manager.io/v1   true         Challenge
orders                             acme.cert-manager.io/v1   true         Order
certificaterequests   cr,crs       cert-manager.io/v1        true         CertificateRequest
certificates          cert,certs   cert-manager.io/v1        true         Certificate
clusterissuers                     cert-manager.io/v1        false        ClusterIssuer
issuers                            cert-manager.io/v1        true         Issuer

  # values.yaml
  labels: {}

- replicaCount: 1
+ replicaCount: 2

  webhookPort: 9443
  syncPeriod: 1m
  ...

# values.yaml
...

authSecret:
  enabled: true
  create: true
  name: "controller-manager"
  annotations: {}
  ### GitHub Apps Configuration
  ## NOTE: IDs MUST be strings, use quotes
  #github_app_id: ""
  #github_app_installation_id: ""
  #github_app_private_key: |
  ### GitHub PAT Configuration
  github_token: "ghp_<YOUR_ADMIN_PAT>"  # Github Actions Runner integration token
  ### Basic auth for github API proxy
  #github_basicauth_username: ""
  #github_basicauth_password: ""

...

helm upgrade \
  --install \
  --namespace actions-runner-system \
  --create-namespace \
  actions-runner-controller actions-runner-controller/actions-runner-controller \
  -f values.yaml \
  --wait

$ helm list -n actions-runner-system
NAME                       NAMESPACE              REVISION  UPDATED                               STATUS    CHART                             APP VERSION
actions-runner-controller  actions-runner-system  6         2023-07-13 03:30:55.747469 +0900 KST  deployed  actions-runner-controller-0.23.3  0.27.4

$ kubectl get pod -n actions-runner-system
NAME                                         READY   STATUS    RESTARTS   AGE
actions-runner-controller-77445d6674-46zgw   2/2     Running   0          76m
actions-runner-controller-77445d6674-v4nw5   2/2     Running   0          100m

kubectl get issuer,certificate \
  -n actions-runner-system \
  -o wide

NAME                                                                 READY   STATUS   AGE
issuer.cert-manager.io/actions-runner-controller-selfsigned-issuer   True             55d

NAME                                                                 READY   SECRET                                   ISSUER                                        STATUS                                          AGE
certificate.cert-manager.io/actions-runner-controller-serving-cert   True    actions-runner-controller-serving-cert   actions-runner-controller-selfsigned-issuer   Certificate is up to date and has not expired   55d

$ kubectl api-resources | grep runner
horizontalrunnerautoscalers       hra          actions.summerwind.dev/v1alpha1        true         HorizontalRunnerAutoscaler
runnerdeployments                 rdeploy      actions.summerwind.dev/v1alpha1        true         RunnerDeployment
runnerreplicasets                 rrs          actions.summerwind.dev/v1alpha1        true         RunnerReplicaSet
runners                                        actions.summerwind.dev/v1alpha1        true         Runner
runnersets                                     actions.summerwind.dev/v1alpha1        true         RunnerSet

# runners.yaml
---
apiVersion: actions.summerwind.dev/v1alpha1
kind: RunnerDeployment
metadata:
  name: test-basic-runner
  namespace: actions-runner
  labels:
    environment: test
    maintainer: younsung.lee
spec:
  template:
    spec:
      enterprise: dogecompany
      labels:
        - t3a.large
        - support-horizontal-runner-autoscaling
        - ubuntu-20.04
        - v2.306.0
      resources:
        # -- Change the value according to your Instance spec.
        limits:
          cpu: "1.5"
          memory: "6Gi"
        # -- Change the value according to your Instance spec.
        requests:
          cpu: "0.5"
          memory: "1Gi"
---
apiVersion: actions.summerwind.dev/v1alpha1
kind: HorizontalRunnerAutoscaler
metadata:
  name: test-basic-runner-autoscaler
  namespace: actions-runner
  labels:
    environment: test
    maintainer: younsung.lee
spec:
  # Runners in the targeted RunnerDeployment won't be scaled down
  # for 5 minutes instead of the default 10 minutes now
  scaleDownDelaySecondsAfterScaleOut: 300
  scaleTargetRef:
    kind: RunnerDeployment
    name: test-basic-runner
  minReplicas: 2
  maxReplicas: 16
  scheduledOverrides:
  # -- 주말에는 Runner 리소스 절약하기
  # minReplicas를 토요일 오전 0시(KST)부터 월요일 오전 0시(KST)까지 0으로 지정
  - startTime: "2023-07-15T00:00:00+09:00"
    endTime: "2023-07-17T00:00:00+09:00"
    recurrenceRule:
      frequency: Weekly
    minReplicas: 1
  metrics:
  - type: PercentageRunnersBusy
    # 사용 중인 Runner의 비율이 75%('0.75')보다 크면 러너 수를 scale-out
    # 사용 중인 Runenr의 비율이 25%('0.25')보다 작으면 러너 수를 scale-in
    scaleUpThreshold: '0.75'
    scaleDownThreshold: '0.25'
    # 증가시킬 때는 현재 러너 수의 '2' 배만큼 증가시키고
    # 감소시킬 때는 현재 러너 수의 절반 '0.5' 만큼 감소
    scaleUpFactor: '2'
    scaleDownFactor: '0.5'

$ kubectl get hra -n actions-runner
NAME                           MIN   MAX   DESIRED   SCHEDULE
test-basic-runner-autoscaler   2     16    1         min=2 time=2023-07-16 15:00:00 +0000 UTC

# runners.yaml
---
apiVersion: actions.summerwind.dev/v1alpha1
kind: RunnerDeployment
...
spec:
  template:
    spec:
      # -- 3 valid runner scope options
      # enterprise: YOUR_ENTERPRISE_NAME
      organization: dogecompany
      # repository: YOUR_SPECIFIC_REPOSITORY_NAME
      ...

  metrics:
  - type: PercentageRunnersBusy
    ...
    scaleUpFactor: '2'
    scaleDownFactor: '0.5'

  metrics:
  - type: PercentageRunnersBusy
    ...
    scaleUpAdujustment: 2   # The scale up runner count added to desired count
    scaleDownAdjustment: 1  # The scale down runner count subtracted from the desired count

# runners.yaml
...
---
apiVersion: actions.summerwind.dev/v1alpha1
kind: HorizontalRunnerAutoscaler
...
spec:
  scaleDownDelaySecondsAfterScaleOut: 300
  ...

$ kubectl create namespace actions-runner

$ kubectl apply -f runners.yaml
runnerdeployment.actions.summerwind.dev/test-basic-runner created
horizontalrunnerautoscaler.actions.summerwind.dev/test-basic-runner-autoscaler created

$ kubectl get runner -n actions-runner

NAME                            ENTERPRISE    ORGANIZATION   REPOSITORY   GROUP   LABELS                                                                            STATUS    MESSAGE   WF REPO   WF RUN   AGE
test-basic-runner-qr6gr-kckjh   dogecompany                                       ["t3a.large","support-horizontal-runner-autoscaling","ubuntu-20.04","v2.306.0"]   Running                                34m
test-basic-runner-qr6gr-pk5kf   dogecompany                                       ["t3a.large","support-horizontal-runner-autoscaling","ubuntu-20.04","v2.306.0"]   Running                                34m

$ kubectl describe runner -n actions-runner test-basic-runner-qr6gr-kckjh

...
Events:
  Type    Reason                    Age   From               Message
  ----    ------                    ----  ----               -------
  Normal  RegistrationTokenUpdated  38m   runner-controller  Successfully update registration token
  Normal  PodCreated                38m   runner-controller  Created pod 'test-basic-runner-mvgqp-wjq94'

name: GitHub Actions Demo
run-name: ${{ github.actor }} is testing out GitHub Actions 🚀
on:
  workflow_dispatch:
jobs:
  Explore-GitHub-Actions:
    runs-on: [self-hosted, linux]
    outputs:
      tag_date: ${{ steps.tag.outputs.date }}
      tag_git_hash: ${{ steps.tag.outputs.git_hash }}
    steps:
      - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
      - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
      - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
      - name: Check out repository code
        uses: actions/checkout@v3
      - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
      - run: echo "🖥️ The workflow is now ready to test your code on the runner."
      - name: List files in the repository
        run: |
          ls ${{ github.workspace }}          
      - run: echo "🍏 This job's status is ${{ job.status }}"

---
apiVersion: actions.summerwind.dev/v1alpha1
kind: RunnerDeployment
metadata:
  name: example-runner-deploy
  namespace: actions-runner
spec:
  template:
    spec:
      organization: dogecompany
+     dockerVolumeMounts:
+       - mountPath: /tmp
+         name: tmp
+     volumeMounts:
+       - mountPath: /tmp
+         name: tmp
+     volumes:
+       - name: tmp
+         emptyDir: {}
      ...

{
    "Version": "2012-10-17",
    "Id": "CrossAccountS3BucketAccessPolicy",
    "Statement": [
        {
            "Sid": "AllowCrossAccountS3BucketAccess",
            "Effect": "Allow",
            "Principal": {
                "AWS": "arn:aws:iam::111122223333:role/actions-runner-role"
            },
            "Action": "s3:*",
            "Resource": [
                "arn:aws:s3:::bucket.dogecompany.com",
                "arn:aws:s3:::bucket.dogecompany.com/*"
            ]
        }
    ]
}

{
    "Version": "2012-10-17",
    "Id": "CrossAccountS3BucketAccessPolicy",
    "Statement": [
        {
            "Sid": "AllowCrossAccountS3BucketAccess",
            "Effect": "Allow",
            "Principal": {
                "AWS": "arn:aws:iam::111122223333:role/actions-runner-role"
            },
            "Action": [
              "s3:ListBucket",
              "s3:GetObject",
              "s3:GetObjectAcl",
              "s3:PutObject",
              "s3:PutObjectAcl"
            ],
            "Resource": [
                "arn:aws:s3:::bucket.dogecompany.com",
                "arn:aws:s3:::bucket.dogecompany.com/*"
            ]
        }
    ]
}

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: actions-runner-role
  namespace: actions-runner
  annotations:
    # -- Service Account에 연결할 IAM Role의 ARN 값을 입력
    eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/actions-runner-role
...

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "AllowAssumeRoleForServiceAccountFromActionsRunnerPod",
      "Effect": "Allow",
      "Principal": {
        "Federated": "arn:aws:iam::111122223333:oidc-provider/oidc.eks.REGION.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041A"
      },
      "Action": "sts:AssumeRoleWithWebIdentity",
      "Condition": {
        "StringEquals": {
          "oidc.eks.REGION.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041A:sub": "system:serviceaccount:actions-runner:actions-runner-role"
        }
      }
    }
  ]
}

---
apiVersion: actions.summerwind.dev/v1alpha1
kind: RunnerDeployment
metadata:
  name: example-runner-deploy
  namespace: actions-runner
spec:
  template:
    spec:
      organization: dogecompany
      # Specify the name of the Service Account that the runner will use
      serviceAccountName: actions-runner-role
      securityContext:
        # -- Use 1000 For Ubuntu 20.04 runner
        #fsGroup: 1000
        # -- Use 1001 for Ubuntu 22.04 runner
        fsGroup: 1001

name: Testing IRSA for docker in docker container
run-name: ${{ github.actor }} is testing out IRSA for docker in docker container 🐋

jobs:
  dind:
    runs-on: [self-hosted, linux]
    steps:
    ...
    - name: Running docker run command in dind
      run: |
        #----------------------------------------------
        # !!! NOTE: dind (docker-in-docker) !!!
        # The `docker run` command is processed by
        # the "docker" container, NOT runner container
        # in the runner pod.
        #----------------------------------------------
        docker run \
          --name dind-test \
          -v /var/run/secrets:/var/run/secrets \
          -e AWS_ROLE_ARN \
          -e AWS_WEB_IDENTITY_TOKEN_FILE \
          ... \
          aws-cli:2.13.5 $YOUR_ARGS