Skip to content

Commit 44fb7e2

Browse files
cliftonccliftonc
committed
Speed up check for updates, get github repos in batches
1 parent 2d63e81 commit 44fb7e2

7 files changed

Lines changed: 137 additions & 50 deletions

File tree

‎action.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ inputs:
2424
description: 'Github repository type (e.g. all|private|internal), defaults to `all`'
2525
repository_filter:
2626
description: 'Regex string to filter repository name by, defaults to .*'
27+
repository_batch_size:
28+
description: 'How many github repos to request in parallel, defaults to 10'
2729
catalog_file:
2830
description: 'Catalog file name to look for in the root of repo, defaults to `catalog_file.yaml`'
2931
outputs:

‎dist/index.js‎

Lines changed: 67 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -20867,14 +20867,34 @@ const loadData = async ({ notion }) => {
2086720867
core.error('Your system table does not contain an "unknown" row!')
2086820868
}
2086920869

20870+
// Get the current service matrix and hashes in bulk to speed up updates
20871+
const services = {}
20872+
const getDatabaseRows = async (startCursor) => {
20873+
const pageRows = await notion.databases.query({
20874+
database_id: database,
20875+
start_cursor: startCursor
20876+
})
20877+
pageRows.results.forEach((item) => {
20878+
const pageId = item.id
20879+
const pageHash = item.properties?.Hash?.rich_text[0]?.text?.content
20880+
const pageName = item.properties?.Name?.title[0]?.text?.content
20881+
services[pageName] = { pageId, pageHash }
20882+
})
20883+
if (pageRows.has_more) {
20884+
return await getDatabaseRows(pageRows.next_cursor)
20885+
}
20886+
}
20887+
await getDatabaseRows()
20888+
2087020889
if (error) {
2087120890
process.exit(1)
2087220891
}
2087320892

2087420893
return {
2087520894
systems,
2087620895
owners,
20877-
structure
20896+
structure,
20897+
services
2087820898
}
2087920899
}
2088020900

@@ -20918,10 +20938,21 @@ const { Octokit } = __nccwpck_require__(7223)
2091820938
const YAML = __nccwpck_require__(9645)
2091920939
const core = __nccwpck_require__(272)
2092020940

20941+
const chunk = (arr, len) => {
20942+
const chunks = []
20943+
let i = 0
20944+
const n = arr.length
20945+
while (i < n) {
20946+
chunks.push(arr.slice(i, i += len))
20947+
}
20948+
return chunks
20949+
}
20950+
2092120951
const getRepos = async () => {
2092220952
const GITHUB_TOKEN = core.getInput('github_token')
2092320953
const repositoryType = core.getInput('repository_type') || 'all'
2092420954
const repositoryFilter = core.getInput('repository_filter') || '.*'
20955+
const repositoryBatchSize = parseInt(core.getInput('repository_batch_size') || '10')
2092520956
const owner = core.getInput('github_owner')
2092620957
const catalogFile = core.getInput('catalog_file') || 'catalog-info.yaml'
2092720958
const repositoryFilterRegex = new RegExp(repositoryFilter)
@@ -20958,6 +20989,7 @@ const getRepos = async () => {
2095820989
core.warning(`Unable to find ${path} in ${repo.name}, not processing`)
2095920990
}
2096020991
}
20992+
core.info(`Completed ${repo.name}`)
2096120993
return repoData
2096220994
}
2096320995

@@ -20986,14 +21018,34 @@ const getRepos = async () => {
2098621018
})
2098721019
core.info(`Using repository filter: ${repositoryFilter}`)
2098821020
core.info(`Found ${repos.length} github repositories, now getting service data for those that match the filter ...`)
20989-
const repoData = []
21021+
21022+
// We will create an array of batches to speed up execution, run each batch
21023+
// In series, and then join them together.
21024+
const repoFns = []
21025+
const repoBatches = []
21026+
2099021027
for (const repo of repos) {
2099121028
if (repo.name.match(repositoryFilterRegex)) {
2099221029
const pushMissing = true
20993-
repoData.push(...await parseServiceDefinition(repo, catalogFile, pushMissing))
21030+
repoFns.push(parseServiceDefinition(repo, catalogFile, pushMissing))
2099421031
}
2099521032
}
2099621033

21034+
// Break into batches
21035+
core.info(`Fetching with batch size of ${repositoryBatchSize} ...`)
21036+
const batchRepos = chunk(repoFns, repositoryBatchSize)
21037+
21038+
// Iterate over those
21039+
for (const batch of batchRepos) {
21040+
core.debug(`Fetching ${batch.length} repos ...`)
21041+
repoBatches.push(await Promise.all(batch))
21042+
}
21043+
21044+
let repoData = await Promise.all(repoBatches)
21045+
21046+
// Now flatten it
21047+
repoData = repoData.flat(2)
21048+
2099721049
// Now we want to sort the repositories based on their name, and the number of dependencies
2099821050
repoData.sort((a, b) => {
2099921051
const aDependsOn = a.spec?.dependsOn?.length || 0
@@ -21268,30 +21320,16 @@ let updatedServices = 0
2126821320
let skippedServices = 0
2126921321
let erroredServices = 0
2127021322

21271-
const updateServices = async (repositories, { notion, database, systems, owners, structure }) => {
21323+
const updateServices = async (repositories, { notion, database, systems, owners, structure, services }) => {
2127221324
for (const repo of repositories) {
2127321325
// Lets see if we can find the row
2127421326
const repoName = repo.metadata?.name || repo._repo.name
21275-
const search = await notion.databases.query({
21276-
database_id: database,
21277-
filter: {
21278-
property: 'Name',
21279-
text: {
21280-
equals: repoName
21281-
}
21282-
}
21283-
})
21284-
21285-
// If we have found any results, lets update
21286-
// If multiple are found we have an issue, but for now
21287-
// Lets just update the first one to not make the problem worse
21288-
if (search.results.length > 0) {
21289-
const pageId = search.results[0].id
21290-
const pageHash = search.results[0].properties?.Hash?.rich_text[0]?.text?.content
21291-
core.debug(`Updating notion info for ${repoName}`)
21327+
// Lets look the service up
21328+
if (services[repoName]) {
21329+
const pageId = services[repoName].pageId
21330+
const pageHash = services[repoName].pageHash
2129221331
await updateNotionRow(repo, pageId, pageHash, { notion, database, systems, owners, structure })
2129321332
} else {
21294-
core.debug(`Creating notion info for ${repoName}`)
2129521333
await createNotionRow(repo, { notion, database, systems, owners, structure })
2129621334
}
2129721335
}
@@ -21306,12 +21344,14 @@ const updateNotionRow = async (repo, pageId, pageHash, { notion, database, syste
2130621344
}
2130721345
const { properties, doUpdate } = createProperties(repo, pageHash, dependsOn, { systems, owners, structure })
2130821346
if (doUpdate) {
21347+
core.debug(`Updating notion info for ${repo._repo.name}`)
2130921348
await notion.pages.update({
2131021349
page_id: pageId,
2131121350
properties
2131221351
})
2131321352
updatedServices++
2131421353
} else {
21354+
core.debug(`Not updating notion info for ${repo._repo.name} as hash unchanged`)
2131521355
skippedServices++
2131621356
}
2131721357
if (repo.metadata?.links) {
@@ -21330,6 +21370,7 @@ const createNotionRow = async (repo, { notion, database, systems, owners, struct
2133021370
dependsOn = await getDependsOn(repo.spec.dependsOn, { notion, database })
2133121371
}
2133221372
const { properties } = createProperties(repo, null, dependsOn, { systems, owners, structure })
21373+
core.debug(`Creating notion info for ${repo._repo.name}`)
2133321374
const page = await notion.pages.create({
2133421375
parent: {
2133521376
database_id: database
@@ -28220,17 +28261,18 @@ try {
2822028261
})
2822128262

2822228263
const refreshData = async () => {
28223-
core.startGroup('Loading systems and owners ...')
28224-
const { systems, owners, structure } = await loadData({ core, notion })
28264+
core.startGroup('Loading services, systems and owners ...')
28265+
const { systems, owners, structure, services } = await loadData({ core, notion })
2822528266
core.info(`Found ${structure.length} fields in the Service database: ${structure.map((item) => item.name)}`)
2822628267
core.info(`Loaded ${Object.keys(systems || {}).length} systems`)
2822728268
core.info(`Loaded ${Object.keys(owners || {}).length} owners`)
28269+
core.info(`Loaded ${Object.keys(services || {}).length} existing services`)
2822828270
core.endGroup()
2822928271
core.startGroup('🌀 Getting github repositories')
2823028272
const repositories = await getRepos({ core })
2823128273
core.endGroup()
2823228274
core.startGroup(`✨ Updating notion with ${repositories.length} services ...`)
28233-
await updateServices(repositories, { core, notion, database, systems, owners, structure })
28275+
await updateServices(repositories, { core, notion, database, systems, owners, structure, services })
2823428276
core.endGroup()
2823528277
}
2823628278

‎src/data.js‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,14 +50,34 @@ const loadData = async ({ notion }) => {
5050
core.error('Your system table does not contain an "unknown" row!')
5151
}
5252

53+
// Get the current service matrix and hashes in bulk to speed up updates
54+
const services = {}
55+
const getDatabaseRows = async (startCursor) => {
56+
const pageRows = await notion.databases.query({
57+
database_id: database,
58+
start_cursor: startCursor
59+
})
60+
pageRows.results.forEach((item) => {
61+
const pageId = item.id
62+
const pageHash = item.properties?.Hash?.rich_text[0]?.text?.content
63+
const pageName = item.properties?.Name?.title[0]?.text?.content
64+
services[pageName] = { pageId, pageHash }
65+
})
66+
if (pageRows.has_more) {
67+
return await getDatabaseRows(pageRows.next_cursor)
68+
}
69+
}
70+
await getDatabaseRows()
71+
5372
if (error) {
5473
process.exit(1)
5574
}
5675

5776
return {
5877
systems,
5978
owners,
60-
structure
79+
structure,
80+
services
6181
}
6282
}
6383

‎src/github.js‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,21 @@ const { Octokit } = require('octokit')
22
const YAML = require('yaml')
33
const core = require('@actions/core')
44

5+
const chunk = (arr, len) => {
6+
const chunks = []
7+
let i = 0
8+
const n = arr.length
9+
while (i < n) {
10+
chunks.push(arr.slice(i, i += len))
11+
}
12+
return chunks
13+
}
14+
515
const getRepos = async () => {
616
const GITHUB_TOKEN = core.getInput('github_token')
717
const repositoryType = core.getInput('repository_type') || 'all'
818
const repositoryFilter = core.getInput('repository_filter') || '.*'
19+
const repositoryBatchSize = parseInt(core.getInput('repository_batch_size') || '10')
920
const owner = core.getInput('github_owner')
1021
const catalogFile = core.getInput('catalog_file') || 'catalog-info.yaml'
1122
const repositoryFilterRegex = new RegExp(repositoryFilter)
@@ -42,6 +53,7 @@ const getRepos = async () => {
4253
core.warning(`Unable to find ${path} in ${repo.name}, not processing`)
4354
}
4455
}
56+
core.info(`Completed ${repo.name}`)
4557
return repoData
4658
}
4759

@@ -70,14 +82,34 @@ const getRepos = async () => {
7082
})
7183
core.info(`Using repository filter: ${repositoryFilter}`)
7284
core.info(`Found ${repos.length} github repositories, now getting service data for those that match the filter ...`)
73-
const repoData = []
85+
86+
// We will create an array of batches to speed up execution, run each batch
87+
// In series, and then join them together.
88+
const repoFns = []
89+
const repoBatches = []
90+
7491
for (const repo of repos) {
7592
if (repo.name.match(repositoryFilterRegex)) {
7693
const pushMissing = true
77-
repoData.push(...await parseServiceDefinition(repo, catalogFile, pushMissing))
94+
repoFns.push(parseServiceDefinition(repo, catalogFile, pushMissing))
7895
}
7996
}
8097

98+
// Break into batches
99+
core.info(`Fetching with batch size of ${repositoryBatchSize} ...`)
100+
const batchRepos = chunk(repoFns, repositoryBatchSize)
101+
102+
// Iterate over those
103+
for (const batch of batchRepos) {
104+
core.debug(`Fetching ${batch.length} repos ...`)
105+
repoBatches.push(await Promise.all(batch))
106+
}
107+
108+
let repoData = await Promise.all(repoBatches)
109+
110+
// Now flatten it
111+
repoData = repoData.flat(2)
112+
81113
// Now we want to sort the repositories based on their name, and the number of dependencies
82114
repoData.sort((a, b) => {
83115
const aDependsOn = a.spec?.dependsOn?.length || 0

‎src/index.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,18 @@ try {
1515
})
1616

1717
const refreshData = async () => {
18-
core.startGroup('Loading systems and owners ...')
19-
const { systems, owners, structure } = await loadData({ core, notion })
18+
core.startGroup('Loading services, systems and owners ...')
19+
const { systems, owners, structure, services } = await loadData({ core, notion })
2020
core.info(`Found ${structure.length} fields in the Service database: ${structure.map((item) => item.name)}`)
2121
core.info(`Loaded ${Object.keys(systems || {}).length} systems`)
2222
core.info(`Loaded ${Object.keys(owners || {}).length} owners`)
23+
core.info(`Loaded ${Object.keys(services || {}).length} existing services`)
2324
core.endGroup()
2425
core.startGroup('🌀 Getting github repositories')
2526
const repositories = await getRepos({ core })
2627
core.endGroup()
2728
core.startGroup(`✨ Updating notion with ${repositories.length} services ...`)
28-
await updateServices(repositories, { core, notion, database, systems, owners, structure })
29+
await updateServices(repositories, { core, notion, database, systems, owners, structure, services })
2930
core.endGroup()
3031
}
3132

‎src/local.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ process.env.INPUT_GITHUB_TOKEN = process.env.GITHUB_TOKEN
77
process.env.INPUT_REPOSITORY_TYPE = 'public'
88
process.env.INPUT_GITHUB_OWNER = 'infinitaslearning'
99
process.env.INPUT_REPOSITORY_FILTER = '.*'
10+
process.env.INPUT_REPOSITORY_BATCH_SIZE = '50'
1011
process.env.INPUT_DATABASE = 'cecaf0beb15945158d155866ff9acce8'
1112
process.env.INPUT_OWNER_DATABASE = '7943615f4dba43b3a3b0f991f4f7136d'
1213
process.env.INPUT_SYSTEM_DATABASE = '121534684fe840a1953500e603c2b602'

‎src/services.js‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,30 +9,16 @@ let updatedServices = 0
99
let skippedServices = 0
1010
let erroredServices = 0
1111

12-
const updateServices = async (repositories, { notion, database, systems, owners, structure }) => {
12+
const updateServices = async (repositories, { notion, database, systems, owners, structure, services }) => {
1313
for (const repo of repositories) {
1414
// Lets see if we can find the row
1515
const repoName = repo.metadata?.name || repo._repo.name
16-
const search = await notion.databases.query({
17-
database_id: database,
18-
filter: {
19-
property: 'Name',
20-
text: {
21-
equals: repoName
22-
}
23-
}
24-
})
25-
26-
// If we have found any results, lets update
27-
// If multiple are found we have an issue, but for now
28-
// Lets just update the first one to not make the problem worse
29-
if (search.results.length > 0) {
30-
const pageId = search.results[0].id
31-
const pageHash = search.results[0].properties?.Hash?.rich_text[0]?.text?.content
32-
core.debug(`Updating notion info for ${repoName}`)
16+
// Lets look the service up
17+
if (services[repoName]) {
18+
const pageId = services[repoName].pageId
19+
const pageHash = services[repoName].pageHash
3320
await updateNotionRow(repo, pageId, pageHash, { notion, database, systems, owners, structure })
3421
} else {
35-
core.debug(`Creating notion info for ${repoName}`)
3622
await createNotionRow(repo, { notion, database, systems, owners, structure })
3723
}
3824
}
@@ -47,12 +33,14 @@ const updateNotionRow = async (repo, pageId, pageHash, { notion, database, syste
4733
}
4834
const { properties, doUpdate } = createProperties(repo, pageHash, dependsOn, { systems, owners, structure })
4935
if (doUpdate) {
36+
core.debug(`Updating notion info for ${repo._repo.name}`)
5037
await notion.pages.update({
5138
page_id: pageId,
5239
properties
5340
})
5441
updatedServices++
5542
} else {
43+
core.debug(`Not updating notion info for ${repo._repo.name} as hash unchanged`)
5644
skippedServices++
5745
}
5846
if (repo.metadata?.links) {
@@ -71,6 +59,7 @@ const createNotionRow = async (repo, { notion, database, systems, owners, struct
7159
dependsOn = await getDependsOn(repo.spec.dependsOn, { notion, database })
7260
}
7361
const { properties } = createProperties(repo, null, dependsOn, { systems, owners, structure })
62+
core.debug(`Creating notion info for ${repo._repo.name}`)
7463
const page = await notion.pages.create({
7564
parent: {
7665
database_id: database

0 commit comments

Comments
 (0)