feat: add standard-site-sync

For now this will be skipped since there's no md files and NPMX_PDS_URL isn't set.

Co-authored-by: Brandon Hurrington <brandon.o.hurrington@gmail.com>

Author: jonathanyeong

modules/standard-site-sync.ts

@@ -0,0 +1,134 @@
+import { readFileSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import { defineNuxtModule, useNuxt, createResolver } from 'nuxt/kit'
+import { safeParse } from 'valibot'
+import * as site from '../shared/types/lexicons/site'
+import { BlogPostSchema } from '../shared/schemas/blog'
+import { NPMX_SITE } from '../shared/utils/constants'
+import { parseBasicFrontmatter } from '../shared/utils/parse-basic-frontmatter'
+import { TID } from '@atproto/common'
+import { Client } from '@atproto/lex'
+
+const syncedDocuments = new Map<string, string>()
+const CLOCK_ID_THREE = 3
+const DATE_TO_MICROSECONDS = 1000
+
+// TODO: Currently logging quite a lot, can remove some later if we want
+export default defineNuxtModule({
+  meta: { name: 'standard-site-sync' },
+  async setup() {
+    const nuxt = useNuxt()
+    const { resolve } = createResolver(import.meta.url)
+    const contentDir = resolve('../app/pages/blog')
+
+    // Authentication with PDS using an app password
+    const pdsUrl = process.env.NPMX_PDS_URL
+    if (!pdsUrl) {
+      console.warn('[standard-site-sync] NPMX_PDS_URL not set, skipping sync')
+      return
+    }
+    // Instantiate a single new client instance that is reused for every file
+    const client = new Client(pdsUrl)
+
+    if (nuxt.options._prepare) return
+
+    nuxt.hook('build:before', async () => {
+      const { globby } = await import('globby')
+      const files: string[] = await globby(`${contentDir}/**/*.md`)
+
+      // INFO: Arbitrarily chosen concurrency limit, can be changed if needed
+      const concurrencyLimit = 5
+      for (let i = 0; i < files.length; i += concurrencyLimit) {
+        const batch = files.slice(i, i + concurrencyLimit)
+        // Process files in parallel
+        await Promise.all(
+          batch.map(file =>
+            syncFile(file, NPMX_SITE, client).catch(error =>
+              console.error(`[standard-site-sync] Error in ${file}:` + error),
+            ),
+          ),
+        )
+      }
+    })
+
+    nuxt.hook('builder:watch', async (event, path) => {
+      if (!path.endsWith('.md')) return
+
+      // Ignore deleted files
+      if (event === 'unlink') {
+        console.log(`[standard-site-sync] File deleted: ${path}`)
+        return
+      }
+
+      // Process add/change events only
+      await syncFile(resolve(nuxt.options.rootDir, path), NPMX_SITE, client).catch(err =>
+        console.error(`[standard-site-sync] Failed ${path}:`, err),
+      )
+    })
+  },
+})
+
+/*
+ * INFO: Loads record to atproto and ensures uniqueness by checking the date the article is published
+ * publishedAt is an id that does not change
+ * Atomicity is enforced with upsert using publishedAt so we always update existing records instead of creating new ones
+ * Clock id(3) provides a deterministic ID
+ * WARN: DOES NOT CATCH ERRORS, THIS MUST BE HANDLED
+ */
+const syncFile = async (filePath: string, siteUrl: string, client: Client) => {
+  const fileContent = readFileSync(filePath, 'utf-8')
+  const frontmatter = parseBasicFrontmatter(fileContent)
+
+  // Schema expects 'path' & frontmatter provides 'slug'
+  const normalizedFrontmatter = {
+    ...frontmatter,
+    path: typeof frontmatter.slug === 'string' ? `/blog/${frontmatter.slug}` : frontmatter.path,
+  }
+
+  const result = safeParse(BlogPostSchema, normalizedFrontmatter)
+  if (!result.success) {
+    console.warn(`[standard-site-sync] Validation failed for ${filePath}`, result.issues)
+    return
+  }
+
+  const data = result.output
+
+  // filter drafts
+  if (data.draft) {
+    if (process.env.DEBUG === 'true') {
+      console.debug(`[standard-site-sync] Skipping draft: ${data.path}`)
+    }
+    return
+  }
+
+  // Keys are sorted to provide a more stable hash
+  const hash = createHash('sha256')
+    .update(JSON.stringify(data, Object.keys(data).sort()))
+    .digest('hex')
+
+  if (syncedDocuments.get(data.path) === hash) {
+    return
+  }
+
+  const document = site.standard.document.$build({
+    site: siteUrl as `${string}:${string}`,
+    path: data.path,
+    title: data.title,
+    description: data.description ?? data.excerpt,
+    tags: data.tags,
+    // This can be extended to update the site.standard.document .updatedAt if it is changed and use the posts date here
+    publishedAt: new Date(data.date).toISOString(),
+  })
+
+  const dateInMicroSeconds = new Date(result.output.date).getTime() * DATE_TO_MICROSECONDS
+
+  // Clock id(3) needs to be the same everytime to get the same TID from a timestamp
+  const tid = TID.fromTime(dateInMicroSeconds, CLOCK_ID_THREE)
+
+  // client.put is async and needs to be awaited
+  await client.put(site.standard.document, document, {
+    rkey: tid.str,
+  })
+
+  syncedDocuments.set(data.path, hash)
+}

package.json

@@ -40,6 +40,7 @@
   },
   "dependencies": {
     "@atproto/api": "^0.18.17",
+    "@atproto/common": "0.5.10",
     "@atproto/lex": "0.0.13",
     "@atproto/oauth-client-node": "^0.3.15",
     "@deno/doc": "jsr:^0.189.1",