fix: use manual pagination to prevent hitting 10k result limit

Replace github.paginate.iterator with manual pagination to have full control over when to stop. This prevents the iterator from hitting GitHub's 10,000 result limit before our code can stop it.
This commit is contained in:
Omer Mishania 2025-11-18 20:08:32 +02:00
parent dad5ef3629
commit 3d29d2e17b

View file

@ -361,39 +361,35 @@ async function findTagByPagination(
repo: string,
tag: string,
): Promise<Release | undefined> {
// Limit pagination to avoid hitting GitHub's 10,000 result limit
// Stop aggressively on empty pages to prevent CI blocking
let pageCount = 0;
// Manually paginate to avoid hitting GitHub's 10,000 result limit
// The github.paginate.iterator can hit the limit before we can stop it
// So we manually paginate with strict limits
const maxPages = 30; // Stop after 30 pages (3000 releases max) to avoid hitting limits
const minPagesBeforeEmptyPageStop = 5; // After checking at least 5 pages, stop immediately on first empty page
const perPage = 100;
// Use the GitHub API directly for manual pagination
const github = (releaser as GitHubReleaser).github;
if (!github) {
// Fallback to iterator if we can't access github directly
let pageCount = 0;
for await (const { data: releases } of releaser.allReleases({
owner,
repo,
})) {
pageCount++;
// Stop if we've checked too many pages
if (pageCount > maxPages) {
console.warn(
`⚠️ Stopped pagination after ${maxPages} pages to avoid hitting GitHub's result limit`,
);
break;
}
// If we get an empty page, stop immediately if we've already checked enough pages
// This prevents getting stuck on empty pages (like pages 300-1000) which blocks CI
if (releases.length === 0) {
if (pageCount >= minPagesBeforeEmptyPageStop) {
if (releases.length === 0 && pageCount >= minPagesBeforeEmptyPageStop) {
console.log(
`Stopped pagination after encountering empty page at page ${pageCount} (to avoid hitting GitHub's result limit)`,
`Stopped pagination after encountering empty page at page ${pageCount}`,
);
break;
}
// If we haven't checked many pages yet, continue (might be at the very end)
continue;
}
const release = releases.find((release) => release.tag_name === tag);
if (release) {
return release;
@ -402,6 +398,64 @@ async function findTagByPagination(
return undefined;
}
// Manual pagination with full control
let page = 1;
let consecutiveEmptyPages = 0;
while (page <= maxPages) {
try {
const response = await github.rest.repos.listReleases({
owner,
repo,
per_page: perPage,
page: page,
});
const releases = response.data;
// If we get an empty page, stop immediately if we've already checked enough pages
if (releases.length === 0) {
consecutiveEmptyPages++;
if (page >= minPagesBeforeEmptyPageStop) {
console.log(
`Stopped pagination after encountering empty page at page ${page} (to avoid hitting GitHub's result limit)`,
);
break;
}
// If we haven't checked many pages yet, continue (might be at the very end)
page++;
continue;
}
// Reset empty page counter when we find releases
consecutiveEmptyPages = 0;
const release = releases.find((release) => release.tag_name === tag);
if (release) {
return release;
}
// If we got fewer results than per_page, we've reached the end
if (releases.length < perPage) {
break;
}
page++;
} catch (error: any) {
// If we hit the 10,000 result limit, stop immediately
if (error.status === 422 && error.message?.includes('10000')) {
console.warn(
`⚠️ Stopped pagination at page ${page} due to GitHub's 10,000 result limit`,
);
break;
}
throw error;
}
}
return undefined;
}
/**
* Finds a release by tag name from all a repository's releases.
*