Skip to content

Commit 06785f2

Browse files
nargokulnargokul
authored
Merge branch 'master' into youngag/iceberg-properties
2 parents 8e21a44 + 404c2b3 commit 06785f2

270 files changed

Lines changed: 36019 additions & 5519 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/_conda-forge-package-release.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
name: _Conda Forge Package Release
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
package:
7+
description: 'Package being released (e.g. sagemaker-train)'
8+
required: true
9+
type: string
10+
feedstock:
11+
description: 'Feedstock repo (e.g. conda-forge/sagemaker-train-feedstock)'
12+
required: true
13+
type: string
14+
pr_search:
15+
description: 'PR title search string (e.g. sagemaker-train v1.6.0)'
16+
required: true
17+
type: string
18+
version:
19+
description: 'Version of this package being released (e.g. 1.6.0)'
20+
required: true
21+
type: string
22+
dep_package:
23+
description: 'Conda dependency to wait for before retrying CI (e.g. sagemaker-core)'
24+
required: true
25+
type: string
26+
dep_version:
27+
description: 'Version of the dependency to wait for (e.g. 2.6.0)'
28+
required: true
29+
type: string
30+
poll_interval:
31+
required: true
32+
type: string
33+
max_attempts:
34+
required: true
35+
type: string
36+
secrets:
37+
token:
38+
required: true
39+
40+
jobs:
41+
release:
42+
runs-on: ubuntu-latest
43+
env:
44+
GH_TOKEN: ${{ secrets.token }}
45+
steps:
46+
- name: Wait for dependency (${{ inputs.dep_package }}==${{ inputs.dep_version }}) on conda-forge
47+
run: |
48+
PACKAGE="${{ inputs.dep_package }}"
49+
VERSION="${{ inputs.dep_version }}"
50+
echo "Waiting for ${PACKAGE}==${VERSION} on conda-forge..."
51+
for i in $(seq 1 ${{ inputs.max_attempts }}); do
52+
RESULT=$(conda search -c conda-forge --override-channels \
53+
"${PACKAGE}==${VERSION}" --json 2>/dev/null \
54+
| python3 -c "import sys,json; d=json.load(sys.stdin); print('found' if d.get('$PACKAGE') else 'not_found')" \
55+
2>/dev/null || echo "not_found")
56+
echo "Attempt $i: ${RESULT}"
57+
[ "$RESULT" = "found" ] && echo "${PACKAGE}==${VERSION} is available." && exit 0
58+
sleep ${{ inputs.poll_interval }}
59+
done
60+
echo "Timed out waiting for ${PACKAGE}==${VERSION}." && exit 1
61+
62+
- name: Merge and wait for ${{ inputs.package }} feedstock PR
63+
run: |
64+
REPO="${{ inputs.feedstock }}"
65+
SEARCH="${{ inputs.pr_search }}"
66+
for i in $(seq 1 ${{ inputs.max_attempts }}); do
67+
STATE=$(gh pr list --repo "$REPO" --state all \
68+
--search "$SEARCH" --json state -q '.[0].state // "NOT_FOUND"')
69+
echo "Attempt $i: ${STATE}"
70+
[ "$STATE" = "MERGED" ] && exit 0
71+
PR=$(gh pr list --repo "$REPO" --state open \
72+
--search "$SEARCH" --json number -q '.[0].number')
73+
if [ -n "$PR" ]; then
74+
CI_STATUS=$(gh pr view "$PR" --repo "$REPO" \
75+
--json statusCheckRollup -q '
76+
.statusCheckRollup | map(
77+
if .__typename == "CheckRun" then .conclusion
78+
elif .__typename == "StatusContext" then .state
79+
else null end
80+
) | if length == 0 then "pending"
81+
elif any(. == null or . == "" or . == "IN_PROGRESS" or . == "QUEUED" or . == "WAITING" or . == "PENDING") then "pending"
82+
elif all(. == "SUCCESS") then "success"
83+
else "failure" end')
84+
echo "CI status: ${CI_STATUS}"
85+
if [ "$CI_STATUS" = "success" ]; then
86+
echo "CI passed, merging PR #${PR}..."
87+
gh pr merge "$PR" --repo "$REPO" --merge 2>/dev/null || true
88+
elif [ "$CI_STATUS" = "failure" ]; then
89+
echo "CI failed, retriggering..."
90+
BRANCH=$(gh pr view "$PR" --repo "$REPO" --json headRefName -q .headRefName)
91+
RUN_ID=$(gh run list --repo "$REPO" --branch "$BRANCH" \
92+
--json databaseId,conclusion -q \
93+
'[.[] | select(.conclusion=="failure")][0].databaseId')
94+
[ -n "$RUN_ID" ] && gh run rerun "$RUN_ID" --repo "$REPO" --failed || true
95+
fi
96+
fi
97+
sleep ${{ inputs.poll_interval }}
98+
done
99+
echo "Timed out waiting for ${{ inputs.package }} PR to merge." && exit 1

‎.github/workflows/conda-forge-release-chain.yml‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
name: Conda Forge Release Chain
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
poll_interval_seconds:
7+
description: 'Seconds between polls (default: 300)'
8+
required: false
9+
default: '300'
10+
timeout_attempts:
11+
description: 'Max poll attempts per package before failing (default: 20 = 100min at 300s)'
12+
required: false
13+
default: '20'
14+
15+
jobs:
16+
read-versions:
17+
runs-on: ubuntu-latest
18+
outputs:
19+
core: ${{ steps.v.outputs.core }}
20+
train: ${{ steps.v.outputs.train }}
21+
serve: ${{ steps.v.outputs.serve }}
22+
mlops: ${{ steps.v.outputs.mlops }}
23+
pysdk: ${{ steps.v.outputs.pysdk }}
24+
meta: ${{ steps.v.outputs.meta }}
25+
steps:
26+
- uses: actions/checkout@v4
27+
- name: Read versions
28+
id: v
29+
run: |
30+
echo "core=$(cat sagemaker-core/VERSION)" >> $GITHUB_OUTPUT
31+
echo "train=$(cat sagemaker-train/VERSION)" >> $GITHUB_OUTPUT
32+
echo "serve=$(cat sagemaker-serve/VERSION)" >> $GITHUB_OUTPUT
33+
echo "mlops=$(cat sagemaker-mlops/VERSION)" >> $GITHUB_OUTPUT
34+
echo "pysdk=$(cat VERSION)" >> $GITHUB_OUTPUT
35+
echo "meta=$(cat VERSION)" >> $GITHUB_OUTPUT
36+
echo "Versions:"
37+
echo " sagemaker-core: $(cat sagemaker-core/VERSION)"
38+
echo " sagemaker-train: $(cat sagemaker-train/VERSION)"
39+
echo " sagemaker-serve: $(cat sagemaker-serve/VERSION)"
40+
echo " sagemaker-mlops: $(cat sagemaker-mlops/VERSION)"
41+
echo " sagemaker-python-sdk: $(cat VERSION)"
42+
echo " sagemaker (meta): $(cat VERSION)"
43+
44+
# sagemaker-train waits for sagemaker-core
45+
release-sagemaker-train:
46+
needs: read-versions
47+
uses: ./.github/workflows/_conda-forge-package-release.yml
48+
with:
49+
package: sagemaker-train
50+
feedstock: conda-forge/sagemaker-train-feedstock
51+
pr_search: "sagemaker-train v${{ needs.read-versions.outputs.train }}"
52+
version: ${{ needs.read-versions.outputs.train }}
53+
dep_package: sagemaker-core
54+
dep_version: ${{ needs.read-versions.outputs.core }}
55+
poll_interval: ${{ github.event.inputs.poll_interval_seconds }}
56+
max_attempts: ${{ github.event.inputs.timeout_attempts }}
57+
secrets:
58+
token: ${{ secrets.CONDA_FORGE_RELEASE }}
59+
60+
# sagemaker-serve waits for sagemaker-train
61+
release-sagemaker-serve:
62+
needs: [read-versions, release-sagemaker-train]
63+
uses: ./.github/workflows/_conda-forge-package-release.yml
64+
with:
65+
package: sagemaker-serve
66+
feedstock: conda-forge/sagemaker-serve-feedstock
67+
pr_search: "sagemaker-serve v${{ needs.read-versions.outputs.serve }}"
68+
version: ${{ needs.read-versions.outputs.serve }}
69+
dep_package: sagemaker-train
70+
dep_version: ${{ needs.read-versions.outputs.train }}
71+
poll_interval: ${{ github.event.inputs.poll_interval_seconds }}
72+
max_attempts: ${{ github.event.inputs.timeout_attempts }}
73+
secrets:
74+
token: ${{ secrets.CONDA_FORGE_RELEASE }}
75+
76+
# sagemaker-mlops waits for sagemaker-serve
77+
release-sagemaker-mlops:
78+
needs: [read-versions, release-sagemaker-serve]
79+
uses: ./.github/workflows/_conda-forge-package-release.yml
80+
with:
81+
package: sagemaker-mlops
82+
feedstock: conda-forge/sagemaker-mlops-feedstock
83+
pr_search: "sagemaker-mlops v${{ needs.read-versions.outputs.mlops }}"
84+
version: ${{ needs.read-versions.outputs.mlops }}
85+
dep_package: sagemaker-serve
86+
dep_version: ${{ needs.read-versions.outputs.serve }}
87+
poll_interval: ${{ github.event.inputs.poll_interval_seconds }}
88+
max_attempts: ${{ github.event.inputs.timeout_attempts }}
89+
secrets:
90+
token: ${{ secrets.CONDA_FORGE_RELEASE }}
91+
92+
# sagemaker-python-sdk waits for sagemaker-mlops
93+
release-sagemaker-python-sdk:
94+
needs: [read-versions, release-sagemaker-mlops]
95+
uses: ./.github/workflows/_conda-forge-package-release.yml
96+
with:
97+
package: sagemaker-python-sdk
98+
feedstock: conda-forge/sagemaker-python-sdk-feedstock
99+
pr_search: "[bot-automerge] sagemaker-python-sdk v${{ needs.read-versions.outputs.pysdk }}"
100+
version: ${{ needs.read-versions.outputs.pysdk }}
101+
dep_package: sagemaker-mlops
102+
dep_version: ${{ needs.read-versions.outputs.mlops }}
103+
poll_interval: ${{ github.event.inputs.poll_interval_seconds }}
104+
max_attempts: ${{ github.event.inputs.timeout_attempts }}
105+
secrets:
106+
token: ${{ secrets.CONDA_FORGE_RELEASE }}
107+
108+
# sagemaker (meta) waits for sagemaker-python-sdk
109+
release-sagemaker:
110+
needs: [read-versions, release-sagemaker-python-sdk]
111+
uses: ./.github/workflows/_conda-forge-package-release.yml
112+
with:
113+
package: sagemaker
114+
feedstock: conda-forge/sagemaker-feedstock
115+
pr_search: "[bot-automerge] sagemaker v${{ needs.read-versions.outputs.meta }}"
116+
version: ${{ needs.read-versions.outputs.meta }}
117+
dep_package: sagemaker-python-sdk
118+
dep_version: ${{ needs.read-versions.outputs.pysdk }}
119+
poll_interval: ${{ github.event.inputs.poll_interval_seconds }}
120+
max_attempts: ${{ github.event.inputs.timeout_attempts }}
121+
secrets:
122+
token: ${{ secrets.CONDA_FORGE_RELEASE }}

‎.github/workflows/fortress-scan.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: Fortress Security Scan
2+
on:
3+
pull_request_target:
4+
branches:
5+
- "master"
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.head_ref }}
9+
cancel-in-progress: true
10+
11+
permissions:
12+
id-token: write
13+
14+
jobs:
15+
collab-check:
16+
runs-on: ubuntu-latest
17+
outputs:
18+
approval-env: ${{ steps.collab-check.outputs.result }}
19+
steps:
20+
- name: Collaborator Check
21+
uses: actions/github-script@v7
22+
id: collab-check
23+
with:
24+
github-token: ${{ secrets.COLLAB_CHECK_TOKEN }}
25+
result-encoding: string
26+
script: |
27+
try {
28+
const res = await github.rest.repos.checkCollaborator({
29+
owner: context.repo.owner,
30+
repo: context.repo.repo,
31+
username: "${{ github.event.pull_request.user.login }}",
32+
});
33+
console.log("Verified ${{ github.event.pull_request.user.login }} is a repo collaborator. Auto Approving.")
34+
return res.status == "204" ? "auto-approve" : "manual-approval"
35+
} catch (error) {
36+
console.log("${{ github.event.pull_request.user.login }} is not a collaborator. Requiring Manual Approval.")
37+
return "manual-approval"
38+
}
39+
40+
wait-for-approval:
41+
runs-on: ubuntu-latest
42+
needs: [collab-check]
43+
environment: ${{ needs.collab-check.outputs.approval-env }}
44+
steps:
45+
- run: echo "Workflow Approved! Starting Fortress Security Scan."
46+
47+
fortress-scan:
48+
runs-on: ubuntu-latest
49+
needs: [wait-for-approval]
50+
steps:
51+
- name: Configure AWS Credentials
52+
uses: aws-actions/configure-aws-credentials@v4
53+
with:
54+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
55+
aws-region: us-west-2
56+
role-duration-seconds: 10800
57+
58+
- name: Run Fortress Security Scan
59+
uses: aws-actions/aws-codebuild-run-build@v1
60+
with:
61+
project-name: ${{ github.event.repository.name }}-ci-fortress-scan
62+
source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,3 +40,5 @@ env/
4040
sagemaker_train/src/**/container_drivers/sm_train.sh
4141
sagemaker_train/src/**/container_drivers/sourcecode.json
4242
sagemaker_train/src/**/container_drivers/distributed.json
43+
.kiro
44+
docs/api/generated/

0 commit comments

Comments
 (0)