Skip to content

Commit 6f5bfca

Browse files
mareklibramareklibra
authored
feat(x2a): add sorting by project status (#2635)
* feat(x2a): add sorting by project status Signed-off-by: Marek Libra <marek.libra@gmail.com> * sliding window pool for maxConcurrency * Disable sorting by status for a higher project list count * simplify sortAndPaginateInMemory() --------- Signed-off-by: Marek Libra <marek.libra@gmail.com>
1 parent b78013f commit 6f5bfca

20 files changed

Lines changed: 1082 additions & 26 deletions

File tree

‎workspaces/x2a/.changeset/purple-otters-turn.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@red-hat-developer-hub/backstage-plugin-x2a-backend': minor
3+
---
4+
5+
Implement sorting by project status.

‎workspaces/x2a/plugins/x2a-backend/src/router/projects.test.ts‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ import { AuthorizeResult } from '@backstage/plugin-permission-common';
2222
import {
2323
createApp,
2424
createDatabase,
25+
createService,
26+
createTestJob,
2527
LONG_TEST_TIMEOUT,
2628
mockInputProject,
2729
nonExistentId,
@@ -500,6 +502,111 @@ describe('createRouter – projects', () => {
500502
LONG_TEST_TIMEOUT,
501503
);
502504

505+
it.each(supportedDatabaseIds)(
506+
'should sort projects by status with correct pagination - %p',
507+
async databaseId => {
508+
const { client } = await createDatabase(databaseId);
509+
const app = await createApp(client);
510+
const x2aDatabase = createService(client);
511+
512+
// Create 3 projects that will get different calculated statuses
513+
const projectA = await request(app)
514+
.post('/projects')
515+
.send({
516+
...mockInputProject,
517+
name: 'Project Alpha',
518+
abbreviation: 'PA',
519+
});
520+
expect(projectA.status).toBe(200);
521+
522+
const projectB = await request(app)
523+
.post('/projects')
524+
.send({
525+
...mockInputProject,
526+
name: 'Project Beta',
527+
abbreviation: 'PB',
528+
});
529+
expect(projectB.status).toBe(200);
530+
531+
const projectC = await request(app)
532+
.post('/projects')
533+
.send({
534+
...mockInputProject,
535+
name: 'Project Charlie',
536+
abbreviation: 'PC',
537+
});
538+
expect(projectC.status).toBe(200);
539+
540+
// Project Alpha: no init job → status.state = 'created'
541+
// Project Beta: init job success → status.state = 'initialized'
542+
// Project Charlie: init job error → status.state = 'failed'
543+
await createTestJob(x2aDatabase, {
544+
projectId: projectB.body.id,
545+
moduleId: null,
546+
phase: 'init',
547+
status: 'success',
548+
});
549+
await createTestJob(x2aDatabase, {
550+
projectId: projectC.body.id,
551+
moduleId: null,
552+
phase: 'init',
553+
status: 'error',
554+
});
555+
556+
// Semantic ascending: created(0) < initialized(2) < failed(4)
557+
const responseAsc = await request(app)
558+
.get('/projects?sort=status&order=asc')
559+
.send();
560+
561+
expect(responseAsc.status).toBe(200);
562+
expect(responseAsc.body.totalCount).toBe(3);
563+
const statesAsc = responseAsc.body.items.map(
564+
(p: { status?: { state: string } }) => p.status?.state,
565+
);
566+
expect(statesAsc).toEqual(['created', 'initialized', 'failed']);
567+
568+
// Semantic descending: failed(4) > initialized(2) > created(0)
569+
const responseDesc = await request(app)
570+
.get('/projects?sort=status&order=desc')
571+
.send();
572+
573+
expect(responseDesc.status).toBe(200);
574+
const statesDesc = responseDesc.body.items.map(
575+
(p: { status?: { state: string } }) => p.status?.state,
576+
);
577+
expect(statesDesc).toEqual(['failed', 'initialized', 'created']);
578+
579+
// Pagination: page 0 with pageSize 2
580+
const page0 = await request(app)
581+
.get('/projects?sort=status&order=asc&pageSize=2&page=0')
582+
.send();
583+
584+
expect(page0.status).toBe(200);
585+
expect(page0.body.totalCount).toBe(3);
586+
expect(page0.body.items).toHaveLength(2);
587+
expect(
588+
page0.body.items.map(
589+
(p: { status?: { state: string } }) => p.status?.state,
590+
),
591+
).toEqual(['created', 'initialized']);
592+
593+
// Pagination: page 1 with pageSize 2
594+
const page1 = await request(app)
595+
.get('/projects?sort=status&order=asc&pageSize=2&page=1')
596+
.send();
597+
598+
expect(page1.status).toBe(200);
599+
expect(page1.body.totalCount).toBe(3);
600+
expect(page1.body.items).toHaveLength(1);
601+
expect(
602+
page1.body.items.map(
603+
(p: { status?: { state: string } }) => p.status?.state,
604+
),
605+
).toEqual(['failed']);
606+
},
607+
LONG_TEST_TIMEOUT,
608+
);
609+
503610
describe('GET /projects and GET /projects/:projectId – x2a.user or x2a admin required', () => {
504611
it.each(supportedDatabaseIds)(
505612
'should deny GET /projects when user has neither x2a.user nor x2a admin view - %p',

‎workspaces/x2a/plugins/x2a-backend/src/router/projects.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ export function registerProjectRoutes(
7171
'createdAt',
7272
'name',
7373
'abbreviation',
74+
// sorting by status is expensive for large datasets
7475
'status',
7576
'description',
7677
'createdBy',

‎workspaces/x2a/plugins/x2a-backend/src/services/X2ADatabaseService/index.ts‎

Lines changed: 108 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,21 @@ import {
3232
MigrationPhase,
3333
Artifact,
3434
Telemetry,
35+
ProjectStatusState,
36+
DEFAULT_PAGE_ORDER,
37+
DEFAULT_PAGE_SIZE,
38+
IN_MEMORY_SORT_WARN_THRESHOLD,
3539
} from '@red-hat-developer-hub/backstage-plugin-x2a-common';
3640

3741
import { ProjectsGet } from '../../schema/openapi';
3842

3943
import { JobOperations, CreateJobInput } from './jobOperations';
4044
import { ModuleOperations } from './moduleOperations';
4145
import { ProjectOperations } from './projectOperations';
46+
import { isNonDbSortField } from './queryHelpers';
4247
import { removeSensitiveFromJob } from '../../router/common';
48+
import { MAX_CONCURRENT_ENRICHMENT_JOBS } from '../constants';
49+
import { maxConcurrency } from '../../utils';
4350
import { calculateModuleStatus, calculateProjectStatus } from './status';
4451

4552
export class X2ADatabaseService {
@@ -104,6 +111,19 @@ export class X2ADatabaseService {
104111
return this.#projectOps.createProject(input, options);
105112
}
106113

114+
/**
115+
* Semantic ordering for ProjectStatusState.
116+
* Lower values appear first in ascending sort.
117+
*/
118+
static readonly STATE_ORDER: Record<ProjectStatusState, number> = {
119+
created: 0,
120+
initializing: 1,
121+
initialized: 2,
122+
inProgress: 3,
123+
failed: 4,
124+
completed: 5,
125+
};
126+
107127
async listProjects(
108128
query: ProjectsGet['query'],
109129
options: {
@@ -112,15 +132,101 @@ export class X2ADatabaseService {
112132
groupsOfUser: string[];
113133
},
114134
): Promise<{ projects: Project[]; totalCount: number }> {
115-
const result = await this.#projectOps.listProjects(query, options);
135+
const sortByComputedField = isNonDbSortField(query.sort);
136+
137+
const result = await this.#projectOps.listProjects(query, options, {
138+
skipPagination: sortByComputedField,
139+
});
140+
141+
if (
142+
sortByComputedField &&
143+
result.totalCount > IN_MEMORY_SORT_WARN_THRESHOLD
144+
) {
145+
// If this proves to be a performance bottleneck, let's consider either
146+
// removing the sort by status
147+
// or having materialized DB column (works with Postgres only)
148+
// or introducing a separate complex DB SELECT for the case of sorting
149+
// by status which will join/group jobs.
150+
this.#logger.warn(
151+
`In-memory sort by "${query.sort}" is loading ${result.totalCount} projects.`,
152+
);
153+
}
116154

117155
this.#logger.info(
118156
`this.#projectOps.listProjects finished, adding migration plans to projects`,
119157
);
120-
await Promise.all(result.projects.map(p => this.enrichProject(p)));
158+
await maxConcurrency(
159+
result.projects.map(p => () => this.enrichProject(p)),
160+
MAX_CONCURRENT_ENRICHMENT_JOBS,
161+
);
162+
163+
if (sortByComputedField) {
164+
this.sortAndPaginateInMemory(result, query);
165+
}
166+
121167
return result;
122168
}
123169

170+
/**
171+
* Sort enriched projects by a computed field and apply pagination in memory.
172+
* Used when the sort field (e.g. "status") has no DB column.
173+
*/
174+
private sortAndPaginateInMemory(
175+
result: { projects: Project[]; totalCount: number },
176+
query: ProjectsGet['query'],
177+
): void {
178+
const order = query.order || DEFAULT_PAGE_ORDER;
179+
180+
if (query.sort === 'status') {
181+
const summaryKeys = [
182+
'finished',
183+
'error',
184+
'running',
185+
'waiting',
186+
'pending',
187+
'cancelled',
188+
] as const;
189+
190+
const sign = order === 'asc' ? 1 : -1;
191+
192+
const stateRank = (p: Project): number =>
193+
X2ADatabaseService.STATE_ORDER[p.status?.state as ProjectStatusState] ??
194+
99;
195+
196+
result.projects.sort((a, b) => {
197+
// Primary: project-level state (created to completed).
198+
const stateCmp = stateRank(a) - stateRank(b);
199+
if (stateCmp !== 0) return sign * stateCmp;
200+
201+
// Secondary: compare module-summary proportions in priority order
202+
// (finished to cancelled) so projects further along sort first.
203+
// Cross-multiplication (a/b vs c/d to a*d vs c*b) avoids floating-point
204+
// division. The sign is preserved because both totals are non-negative.
205+
const sumA = a.status?.modulesSummary;
206+
const sumB = b.status?.modulesSummary;
207+
const totalA = sumA?.total || 0;
208+
const totalB = sumB?.total || 0;
209+
210+
for (const key of summaryKeys) {
211+
const diff =
212+
(sumA?.[key] ?? 0) * totalB - (sumB?.[key] ?? 0) * totalA;
213+
if (diff !== 0) return sign * diff;
214+
}
215+
216+
return 0;
217+
});
218+
} else {
219+
this.#logger.error(
220+
`No in-memory sort implementation for computed field "${query.sort}", result will be unsorted.`,
221+
);
222+
}
223+
224+
const pageSize = query.pageSize || DEFAULT_PAGE_SIZE;
225+
const page = query.page || 0;
226+
const start = page * pageSize;
227+
result.projects = result.projects.slice(start, start + pageSize);
228+
}
229+
124230
/**
125231
* Use skipEnrichment to avoid fetching the migration plan and status for a project.
126232
*/

‎workspaces/x2a/plugins/x2a-backend/src/services/X2ADatabaseService/projectOperations.ts‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -100,15 +100,14 @@ export class ProjectOperations {
100100
canViewAll?: boolean;
101101
groupsOfUser: string[];
102102
},
103+
dbOptions?: { skipPagination?: boolean },
103104
): Promise<{ projects: Project[]; totalCount: number }> {
104105
const calledByUserRef = getUserRef(options.credentials);
105106
this.#logger.info(`listProjects called by ${calledByUserRef}`);
106107

107108
const pageSize = query.pageSize || DEFAULT_PAGE_SIZE;
108109

109-
const rows = await this.#dbClient('projects')
110-
.limit(pageSize)
111-
.offset((query.page || 0) * pageSize)
110+
const rowsQuery = this.#dbClient('projects')
112111
.select('*')
113112
.modify(queryBuilder =>
114113
filterPermissions(
@@ -117,35 +116,54 @@ export class ProjectOperations {
117116
calledByUserRef,
118117
options.groupsOfUser,
119118
),
120-
)
121-
.orderBy(
122-
mapSortToDatabaseColumn(query.sort) || DEFAULT_PAGE_SORT,
123-
query.order || DEFAULT_PAGE_ORDER,
124119
);
125120

126-
const totalCount = (await this.#dbClient('projects')
127-
.count('*', { as: 'count' })
128-
.modify(queryBuilder =>
129-
filterPermissions(
130-
queryBuilder,
131-
options.canViewAll,
132-
calledByUserRef,
133-
options.groupsOfUser,
134-
),
135-
)
136-
.first()) as { count: string | number };
121+
if (!dbOptions?.skipPagination) {
122+
rowsQuery
123+
.orderBy(
124+
mapSortToDatabaseColumn(query.sort) || DEFAULT_PAGE_SORT,
125+
query.order || DEFAULT_PAGE_ORDER,
126+
)
127+
.limit(pageSize)
128+
.offset((query.page || 0) * pageSize);
129+
}
130+
131+
const rows = await rowsQuery;
132+
133+
// When pagination is skipped, rows already contains every matching row,
134+
// so a separate COUNT query would be redundant.
135+
const totalCount = dbOptions?.skipPagination
136+
? rows.length
137+
: Number.parseInt(
138+
String(
139+
(
140+
(await this.#dbClient('projects')
141+
.count('*', { as: 'count' })
142+
.modify(queryBuilder =>
143+
filterPermissions(
144+
queryBuilder,
145+
options.canViewAll,
146+
calledByUserRef,
147+
options.groupsOfUser,
148+
),
149+
)
150+
.first()) as { count: string | number }
151+
)?.count,
152+
),
153+
10,
154+
);
137155

138156
const projects: Project[] = rows.map((row: Record<string, unknown>) =>
139157
mapRowToProject(row),
140158
);
141159

142160
this.#logger.debug(
143-
`Fetched ${projects.length} out of ${totalCount.count} projects from database (permissions applied)`,
161+
`Fetched ${projects.length} out of ${totalCount} projects from database (permissions applied)`,
144162
);
145163

146164
return {
147165
projects,
148-
totalCount: Number.parseInt(String(totalCount.count), 10),
166+
totalCount,
149167
};
150168
}
151169

0 commit comments

Comments
 (0)