fix: stop immediately on empty pages to prevent iterating through hundreds of empty pages

When encountering empty pages (like pages 300-1000), stop immediately instead of continuing to iterate. This prevents hitting GitHub's 10k result limit when there are many empty pages in the middle of pagination.
This commit is contained in:
Omer Mishania 2025-11-18 20:09:21 +02:00
parent 3d29d2e17b
commit 08c3b6be76

View file

@ -364,15 +364,17 @@ async function findTagByPagination(
// Manually paginate to avoid hitting GitHub's 10,000 result limit // Manually paginate to avoid hitting GitHub's 10,000 result limit
// The github.paginate.iterator can hit the limit before we can stop it // The github.paginate.iterator can hit the limit before we can stop it
// So we manually paginate with strict limits // So we manually paginate with strict limits
// Stop immediately on empty pages to avoid iterating through hundreds of empty pages
const maxPages = 30; // Stop after 30 pages (3000 releases max) to avoid hitting limits const maxPages = 30; // Stop after 30 pages (3000 releases max) to avoid hitting limits
const minPagesBeforeEmptyPageStop = 5; // After checking at least 5 pages, stop immediately on first empty page
const perPage = 100; const perPage = 100;
// Use the GitHub API directly for manual pagination // Use the GitHub API directly for manual pagination
const github = (releaser as GitHubReleaser).github; const github = (releaser as GitHubReleaser).github;
if (!github) { if (!github) {
// Fallback to iterator if we can't access github directly // Fallback to iterator if we can't access github directly
// Stop immediately on empty pages to avoid iterating through hundreds of empty pages
let pageCount = 0; let pageCount = 0;
let foundAnyReleases = false;
for await (const { data: releases } of releaser.allReleases({ for await (const { data: releases } of releaser.allReleases({
owner, owner,
repo, repo,
@ -384,12 +386,18 @@ async function findTagByPagination(
); );
break; break;
} }
if (releases.length === 0 && pageCount >= minPagesBeforeEmptyPageStop) { // Stop immediately on empty pages if we've found releases before
console.log( if (releases.length === 0) {
`Stopped pagination after encountering empty page at page ${pageCount}`, if (foundAnyReleases || pageCount > 1) {
); console.log(
break; `Stopped pagination after encountering empty page at page ${pageCount} (to avoid iterating through empty pages)`,
);
break;
}
// Page 1 is empty, no releases exist
return undefined;
} }
foundAnyReleases = true;
const release = releases.find((release) => release.tag_name === tag); const release = releases.find((release) => release.tag_name === tag);
if (release) { if (release) {
return release; return release;
@ -399,8 +407,9 @@ async function findTagByPagination(
} }
// Manual pagination with full control // Manual pagination with full control
// Stop immediately on empty pages to avoid iterating through hundreds of empty pages
let page = 1; let page = 1;
let consecutiveEmptyPages = 0; let foundAnyReleases = false;
while (page <= maxPages) { while (page <= maxPages) {
try { try {
@ -413,22 +422,21 @@ async function findTagByPagination(
const releases = response.data; const releases = response.data;
// If we get an empty page, stop immediately if we've already checked enough pages // If we get an empty page:
// - If we've found releases before, stop immediately (we've hit a gap or the end)
// - If page 1 is empty, that's fine (no releases exist), return undefined
if (releases.length === 0) { if (releases.length === 0) {
consecutiveEmptyPages++; if (foundAnyReleases || page > 1) {
if (page >= minPagesBeforeEmptyPageStop) {
console.log( console.log(
`Stopped pagination after encountering empty page at page ${page} (to avoid hitting GitHub's result limit)`, `Stopped pagination after encountering empty page at page ${page} (to avoid iterating through empty pages)`,
); );
break; break;
} }
// If we haven't checked many pages yet, continue (might be at the very end) // Page 1 is empty, no releases exist
page++; return undefined;
continue;
} }
// Reset empty page counter when we find releases foundAnyReleases = true;
consecutiveEmptyPages = 0;
const release = releases.find((release) => release.tag_name === tag); const release = releases.find((release) => release.tag_name === tag);
if (release) { if (release) {