Allow system, segment and team to be linked to other databases

Author: cliftonc

action.yml

@@ -16,6 +16,12 @@ inputs:
   database:
     description: 'Parent database to add to'
     required: true
+  system_database:
+    description: 'Database ID to read systems from'    
+  segment_database:
+    description: 'Database ID to read segments from'    
+  team_database:
+    description: 'Database ID to read teams from'    
   repository_type:
     description: 'Github repository type (e.g. all|private|internal), defaults to `all`' 
   catalog_file:

catalog-info.yaml

@@ -5,7 +5,7 @@ metadata:
   description: A githb action to keep a notion database in sync with Github repository info  
   annotations:
     segment: general
-    team: all
+    team: opensource
   tags:
     - notion
     - github

readme.md

@@ -8,9 +8,9 @@ First, you need to have an integration access token - which you can get from htt
 
 By default integrations cant access any content so you you *must* share your database (or the parent page / tree it is contained within) with the integration you created earlier to be able to access it.
 
-## Notion Database
+## Notion Databases
 
-This action expects a Notion database with the following properties:
+This action expects a Notion database with the following properties, this will become the 
 
   - Name: text
   - Description: text
@@ -30,6 +30,19 @@ It looks like this after it has run:
 
 <img width="1451" alt="Screenshot 2021-12-19 at 12 55 39" src="https://user-images.githubusercontent.com/239305/146673989-01187d53-d2fd-42ba-9968-31442b8cc92d.png">
 
+## Relation Databases
+
+You have an option to provide additional 'lookup' databases to convert some of the above selects into relations:
+
+### Segment, Team, System
+
+1. Create a database that has *at least* a `Name` column that is unique, all other columns are up to you.
+2. Create an 'Unknown' row - this is what services that cannot be mapped go to (this name matters).
+3. In the config of the action, provide the input variable `segment_database` pointing to this database.
+4. In the main service catalogue table convert the `Segment` column to a relation, pointing at the above database.
+
+This can be repeated for `Team` and `System`.  The full config is below in the usage.
+
 ## Service Descriptor Format
 
 This action expects each of your repositories to have a descriptor file format in the root of the repo in the form of a Backstage `catalog-info` file.  This is because we are testing this approach against using Backstage directly, and wanted to leverage a format that perhaps has a chance of becoming a defacto standard.  It does not currently map all fields from Backstage, but if you look at the code you can see what it does map.  I may add a config option to allow mapping to be more dynamic in future (a good PR!). 
@@ -57,6 +70,9 @@ jobs:
          github_token: ${{ secrets.PAT_GITHUB_TOKEN }}
          notion_token: ${{ secrets.NOTION_TOKEN }}
          database: 2b26b4290cc84d95ad3e93c3255277a1    
+         segment_database: '7943615f4dba43b3a3b0f991f4f7136d'
+         team_database: 'c11736fe61b941149de098676bde8d92'
+         system_database: '121534684fe840a1953500e603c2b602'
          repository_type: all
          catalog_file: catalog-info.yaml
 

src/index.js

@@ -7,6 +7,9 @@ try {
   const NOTION_TOKEN = core.getInput('notion_token')
   const GITHUB_TOKEN = core.getInput('github_token')
   const database = core.getInput('database')
+  const systemDb = core.getInput('system_database')
+  const segmentDb = core.getInput('segment_database')
+  const teamDb = core.getInput('team_database')
   const owner = core.getInput('github_owner')
   const catalogFile = core.getInput('catalog_file') || 'catalog-info.yaml'
   const repositoryType = core.getInput('repository_type') || 'all'
@@ -17,8 +20,6 @@ try {
     logLevel: LogLevel.ERROR
   })
 
-  console.log(database, owner)
-
   const octokit = new Octokit({ auth: GITHUB_TOKEN })
 
   const getRepos = async () => {
@@ -55,7 +56,60 @@ try {
     return repoData
   }
 
-  const createProperties = (repo) => {
+  const createProperties = (repo, { systems, segments, teams }) => {
+    let segment, team, system
+    const segmentAnnotation = repo?.metadata?.annotations?.segment
+    const systemAnnotation = repo?.metadata?.annotations?.system
+    const teamAnnotation = repo?.metadata?.annotations?.team
+
+    if (segments) {
+      // Segments are a relation
+      segment = {
+        relation: [
+          { id: segments[segmentAnnotation?.toLowerCase()] || segments.unknown }
+        ]
+      }
+    } else {
+      // Segments are a tag
+      segment = {
+        select: {
+          name: segmentAnnotation || 'Unknown'
+        }
+      }
+    }
+
+    if (teams) {
+      // Teams are a relation
+      team = {
+        relation: [
+          { id: teams[teamAnnotation?.toLowerCase()] || teams.unknown }
+        ]
+      }
+    } else {
+      // Teams are a tag
+      team = {
+        select: {
+          name: teamAnnotation || 'Unknown'
+        }
+      }
+    }
+
+    if (systems) {
+      // Segments are a relation
+      system = {
+        relation: [
+          { id: systems[systemAnnotation?.toLowerCase()] || systems.unknown }
+        ]
+      }
+    } else {
+      // Segments are a tag
+      system = {
+        select: {
+          name: systemAnnotation || 'Unknown'
+        }
+      }
+    }
+
     return {
       Name: {
         title: [
@@ -83,16 +137,9 @@ try {
       URL: {
         url: repo._repo.html_url
       },
-      Segment: {
-        select: {
-          name: repo?.metadata?.annotations?.segment || 'Unknown'
-        }
-      },
-      Team: {
-        select: {
-          name: repo?.metadata?.annotations?.team || 'Unknown'
-        }
-      },
+      Segment: segment,
+      Team: team,
+      System: system,
       Visibility: {
         select: {
           name: repo._repo.visibility
@@ -119,31 +166,31 @@ try {
     }
   }
 
-  const updateNotionRow = async (repo, pageId) => {
+  const updateNotionRow = async (repo, pageId, { systems, segments, teams }) => {
     try {
       await notion.pages.update({
         page_id: pageId,
-        properties: createProperties(repo)
+        properties: createProperties(repo, { systems, segments, teams })
       })
     } catch (ex) {
       core.error(`Error updating notion document for ${repo._repo.name}: ${ex.message} ...`)
     }
   }
 
-  const createNotionRow = async (repo) => {
+  const createNotionRow = async (repo, { systems, segments, teams }) => {
     try {
       await notion.pages.create({
         parent: {
           database_id: database
         },
-        properties: createProperties(repo)
+        properties: createProperties(repo, { systems, segments, teams })
       })
     } catch (ex) {
       core.error(`Error creating notion document for ${repo._repo.name}: ${ex.message} ...`)
     }
   }
 
-  const updateNotion = async (repositories) => {
+  const updateNotion = async (repositories, { systems, segments, teams }) => {
     for (const repo of repositories) {
       // Lets see if we can find the row
       const search = await notion.databases.query({
@@ -161,19 +208,62 @@ try {
       // Lets just update the first one to not make the problem worse
       if (search.results.length > 0) {
         const pageId = search.results[0].id
-        await updateNotionRow(repo, pageId)
+        await updateNotionRow(repo, pageId, { systems, segments, teams })
       } else {
-        await createNotionRow(repo)
+        await createNotionRow(repo, { systems, segments, teams })
       }
     }
   }
 
+  const loadData = async () => {
+    const processRows = (data) => {
+      const parent = {}
+      data.results.forEach((row) => {
+        const name = row.properties.Name.title[0].plain_text.toLowerCase()
+        if (name) parent[name] = row.id
+      })
+      return parent
+    }
+
+    let systemRows, segmentRows, teamRows
+
+    if (systemDb) {
+      systemRows = await notion.databases.query({
+        database_id: systemDb
+      })
+    }
+
+    if (segmentDb) {
+      segmentRows = await notion.databases.query({
+        database_id: segmentDb
+      })
+    }
+
+    if (teamDb) {
+      teamRows = await notion.databases.query({
+        database_id: teamDb
+      })
+    }
+
+    return {
+      systems: systemDb ? processRows(systemRows) : null,
+      segments: segmentDb ? processRows(segmentRows) : null,
+      teams: teamDb ? processRows(teamRows) : null
+    }
+  }
+
   const refreshData = async () => {
+    core.startGroup('Loading systems, segments and teams')
+    const { systems, segments, teams } = await loadData()
+    core.info(`Loaded ${Object.keys(systems || {}).length} systems`)
+    core.info(`Loaded ${Object.keys(segments || {}).length} segments`)
+    core.info(`Loaded ${Object.keys(teams || {}).length} teams`)
+    core.endGroup()
     core.startGroup('🌀 Getting github repositories')
     const repositories = await getRepos()
     core.endGroup()
     core.startGroup(`✨ Updating notion with ${repositories.length} services ...`)
-    await updateNotion(repositories)
+    await updateNotion(repositories, { systems, segments, teams })
     core.endGroup()
   }
 

src/index.test.js

@@ -10,6 +10,10 @@ test('complete input should succeed with default inputs', () => {
   process.env.INPUT_GITHUB_TOKEN = process.env.GITHUB_TOKEN
   process.env.INPUT_REPOSITORY_TYPE = 'public'
   process.env.INPUT_GITHUB_OWNER = 'infinitaslearning'
+  process.env.INPUT_SEGMENT_DATABASE = '7943615f4dba43b3a3b0f991f4f7136d'
+  process.env.INPUT_TEAM_DATABASE = 'c11736fe61b941149de098676bde8d92'
+  process.env.INPUT_SYSTEM_DATABASE = '121534684fe840a1953500e603c2b602'
+
   const ip = path.join(__dirname, 'index.js')
   const options = {
     env: process.env

src/local.js

@@ -2,11 +2,14 @@ const { spawn } = require('child_process')
 const path = require('path')
 const process = require('process')
 
-process.env.INPUT_DATABASE = '2b26b4290cc84d95ad3e93c3255277a1' // 'cecaf0beb15945158d155866ff9acce8'
+process.env.INPUT_DATABASE = 'cecaf0beb15945158d155866ff9acce8' // '2b26b4290cc84d95ad3e93c3255277a1'
 process.env.INPUT_NOTION_TOKEN = process.env.NOTION_TOKEN
 process.env.INPUT_GITHUB_TOKEN = process.env.GITHUB_TOKEN
-process.env.INPUT_REPOSITORY_TYPE = 'all'
+process.env.INPUT_REPOSITORY_TYPE = 'public'
 process.env.INPUT_GITHUB_OWNER = 'infinitaslearning'
+process.env.INPUT_SEGMENT_DATABASE = '7943615f4dba43b3a3b0f991f4f7136d'
+process.env.INPUT_TEAM_DATABASE = 'c11736fe61b941149de098676bde8d92'
+process.env.INPUT_SYSTEM_DATABASE = '121534684fe840a1953500e603c2b602'
 
 const ip = path.join(__dirname, 'index.js')
 const options = {