Build system rework (#2169)

* refactor(crypto): remove circular dependency

* refactor(crypto): expose compress/decompress as part of the DidKeyPlugin interface

* fix(crypto): remove import from private file

* refactor: isolate tsconfig

* fix: remove unused bench file

* chore(repo): remove unused deps

* fix(ozone): properly list dependencies

* fix(services): do lint js files

* fix(services/pds): remove unused deps

* chore(pds): remove bench

* chore(dev-env): remove unused deps

* chore(api): remove bench

* remove unused babel.config.js files

* fix: remove .ts extension from import

* fix(pds): remove imports of src files

* fix(tsconfig): properly list all projects

* fix(dev-env): remove imports of src files

* fix(bsky): remove direct import to crypto src

* fix(api): remove imports to api internals

* chore(build): prevent bundling of built output

* chore(dev): add "dev" script to build in watch mode

* chore(deps): move ts-node dependency where it is actually used

* fix(deps): add dev-env as project dependency

* fix(xrpc-server): properly type kexicon

* fix(bsky): improve typings

* fix(pds): fully type formatRecordEmbedInternal return value

* fix(repo): remove imports from @ipld/car/api

* feat(dev-env): re-export BskyIngester

* fix: properly lint & type jest config & test files

* fix(ci): test after build

* fix(types): use NodeJS.Timeout instead of NodeJS.Timer

* fix(bsky): make types exportable

* fix(ozone): make types exportable

* fix(xrpc-server): make types exportable

* fix(xprc-server): make code compliant with "node" types

* fix(xrpc-server): avoid accessing properties of unknown

* chore(deps): update @types/node

* feat(tsconfig): narrow down available types depending on the package's target environment

* fix(pds): remove unused prop

* fix(bsync): Database's migrator not always initialized

* fix(dev-env): remove unreachable code

* fix(xrpc-server): remove unused import

* fix(xrpc-server): mark header property as abstract

* fix(pds): initialize LeakyTxPlugin's txOver property

* fix(bsky): initialize LeakyTxPlugin's txOver property

* fix(bsky): remove unused migrator from DatabaseCoordinator

* fix(bsky): Properly initialize LabelService's cache property

* fix(ozone): Database's migrator not initialized

* fix(ozone): initialize LeakyTxPlugin's txOver property

* fix(crypto): ignore unused variable error

* feat(tsconfig): use stricter rules

* feat(tsconfig): enable useDefineForClassFields

* feat(xrpc-server): add support for brotli incoming payload

* fix(xrpc-server): properly parse & process content-encoding

* fix(common:stream): always call cb in _transform

* tidy/fix tests and service entrypoints

* Revert "fix(xrpc-server): properly parse & process content-encoding"

This reverts commit 2b1c66e153820d3e128fc839fcc1834d52a66686.

* Revert "feat(xrpc-server): add support for brotli incoming payload"

This reverts commit e710c21e6118214ddf215b0515e68cb87299a952.

* remove special node env for tests (defaults to jest val of "test")

* kill mute sync handler on disconnect

* work around connect-es bug w/ request aborts

* style(crypto): rename imports from uint8arrays

* fix update package-lock

* fix lint

* force hbs files to be bundled as cjs

* fix: use concurrently instead of npm-run-all

npm-run-all seems not to be maintained anymore. Additionally, concurrently better forwards signals to child processes.

* remove concurrently alltogether

* ignore sqlite files in services/pds

* fix verify

* fix verify

* tidy, fix verify

* fix blob diversion test

* build rework changeset

---------

Co-authored-by: Devin Ivy <devinivy@gmail.com>

.changeset/long-planes-bathe.md

@@ -0,0 +1,21 @@
+---
+'@atproto/xrpc-server': minor
+'@atproto/common-web': minor
+'@atproto/identity': minor
+'@atproto/dev-env': minor
+'@atproto/lex-cli': minor
+'@atproto/lexicon': minor
+'@atproto/common': minor
+'@atproto/crypto': minor
+'@atproto/syntax': minor
+'@atproto/repo': minor
+'@atproto/xrpc': minor
+'@atproto/api': minor
+'@atproto/aws': minor
+'@atproto/bsync': patch
+'@atproto/ozone': patch
+'@atproto/bsky': patch
+'@atproto/pds': patch
+---
+
+Build system rework, stop bundling dependencies.

.eslintrc

@@ -13,16 +13,7 @@
     "plugin:prettier/recommended",
     "prettier"
   ],
-  "ignorePatterns": [
+  "ignorePatterns": ["dist", "node_modules"],
-    "dist",
-    "node_modules",
-    "jest.config.base.js",
-    "jest.bench.config.js",
-    "jest.config.js",
-    "babel.config.js",
-    "build.js",
-    "update-pkg.js"
-  ],
   "rules": {
     "no-var": "error",
     "prefer-const": "warn",
@@ -36,5 +27,21 @@
     "@typescript-eslint/explicit-module-boundary-types": "off",
     "@typescript-eslint/no-empty-function": "off",
     "@typescript-eslint/no-explicit-any": "off"
-  }
+  },
+  "overrides": [
+    {
+      "files": ["jest.config.js"],
+      "env": { "commonjs": true }
+    },
+    {
+      "files": ["jest.setup.js"],
+      "env": { "jest": true }
+    },
+    {
+      "files": "*.js",
+      "rules": {
+        "@typescript-eslint/no-var-requires": "off"
+      }
+    }
+  ]
 }

.github/workflows/publish.yaml

@@ -27,6 +27,7 @@ jobs:
           node-version: 18
           cache: 'pnpm'
       - run: pnpm i --frozen-lockfile
+      - run: pnpm build
       - run: pnpm verify
       - name: Publish
         id: changesets

.github/workflows/repo.yaml

@@ -24,8 +24,14 @@ jobs:
           cache: 'pnpm'
       - run: pnpm i --frozen-lockfile
       - run: pnpm build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: packages/*/dist
+          retention-days: 1
   test:
     name: Test
+    needs: build
     strategy:
       matrix:
         shard: [1/8, 2/8, 3/8, 4/8, 5/8, 6/8, 7/8, 8/8]
@@ -40,9 +46,14 @@ jobs:
           node-version: 18
           cache: 'pnpm'
       - run: pnpm i --frozen-lockfile
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: packages
       - run: pnpm test:withFlags --maxWorkers=1 --shard=${{ matrix.shard }} --passWithNoTests
   verify:
     name: Verify
+    needs: build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -54,4 +65,8 @@ jobs:
           node-version: 18
           cache: 'pnpm'
       - run: pnpm install --frozen-lockfile
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: packages
       - run: pnpm verify

babel.config.js

@@ -1,3 +0,0 @@
-module.exports = {
-  presets: [['@babel/preset-env']],
-}

jest.config.base.js

@@ -1,20 +0,0 @@
-// Jest doesn't like ES modules, so we need to transpile them
-// For each one, add them to this list, add them to
-// "workspaces.nohoist" in the root package.json, and
-// make sure that a babel.config.js is in the package root
-const esModules = ['get-port', 'node-fetch'].join('|')
-
-// jestconfig.base.js
-module.exports = {
-  roots: ['<rootDir>/src', '<rootDir>/tests'],
-  transform: {
-    '^.+\\.(t|j)s?$': '@swc/jest',
-    '^.+\\.hbs$': require.resolve('handlebars-jest'),
-  },
-  transformIgnorePatterns: [`<rootDir>/node_modules/(?!${esModules})`],
-  testRegex: '(/tests/.*.(test|spec)).(jsx?|tsx?)$',
-  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
-  setupFiles: ['<rootDir>/../../test-setup.ts'],
-  verbose: true,
-  testTimeout: 60000,
-}

jest.config.js

@@ -1,7 +1,4 @@
-// jest.config.js
+/** @type {import('jest').Config} */
-const base = require('./jest.config.base.js')
-
 module.exports = {
-  ...base,
   projects: ['<rootDir>/packages/*/jest.config.js'],
 }

package.json

@@ -10,44 +10,46 @@
   },
   "scripts": {
     "lint:fix": "pnpm lint --fix",
-    "lint": "eslint . --ext .ts,.tsx",
+    "lint": "eslint . --ext .ts,.js",
-    "verify": "prettier --check . && pnpm lint",
+    "style:fix": "prettier --write .",
-    "format": "prettier --write .",
+    "style": "prettier --check .",
-    "build": "pnpm -r --stream build",
+    "verify": "pnpm --stream '/^verify:.+$/'",
-    "update-main-to-dist": "pnpm -r --stream update-main-to-dist",
+    "verify:style": "pnpm run style",
-    "test": "LOG_ENABLED=false NODE_ENV=development ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test",
+    "verify:lint": "pnpm lint",
-    "test:withFlags": "LOG_ENABLED=false NODE_ENV=development ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test --",
+    "verify:types": "tsc --build tsconfig.json",
+    "format": "pnpm lint:fix && pnpm style:fix",
+    "build": "pnpm --recursive --stream build",
+    "dev": "pnpm --stream '/^dev:.+$/'",
+    "dev:tsc": "tsc --build tsconfig.json --watch",
+    "dev:pkg": "pnpm --recursive --parallel --stream dev",
+    "test": "LOG_ENABLED=false ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test",
+    "test:withFlags": "LOG_ENABLED=false ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test --",
     "changeset": "changeset",
     "release": "pnpm build && changeset publish",
     "version-packages": "changeset version && git add ."
   },
   "devDependencies": {
+    "@atproto/dev-env": "workspace:^",
     "@babel/core": "^7.18.6",
     "@babel/preset-env": "^7.18.6",
     "@changesets/changelog-github": "^0.4.8",
     "@changesets/cli": "^2.26.2",
-    "@npmcli/package-json": "^3.0.0",
     "@swc/core": "^1.3.42",
     "@swc/jest": "^0.2.24",
     "@types/jest": "^28.1.4",
-    "@types/node": "^18.0.0",
+    "@types/node": "^18.19.24",
     "@typescript-eslint/eslint-plugin": "^6.14.0",
     "@typescript-eslint/parser": "^6.14.0",
     "babel-eslint": "^10.1.0",
     "dotenv": "^16.0.3",
-    "esbuild": "^0.14.48",
-    "esbuild-node-externals": "^1.5.0",
-    "esbuild-plugin-handlebars": "^1.0.2",
     "eslint": "^8.24.0",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-prettier": "^4.2.1",
-    "handlebars-jest": "^1.0.0",
     "jest": "^28.1.2",
     "node-gyp": "^9.3.1",
     "pino-pretty": "^9.1.0",
     "prettier": "^2.7.1",
     "prettier-config-standard": "^5.0.0",
-    "ts-node": "^10.8.2",
     "typescript": "^5.3.3"
   },
   "workspaces": {

packages/api/babel.config.js

@@ -1,3 +0,0 @@
-module.exports = {
-  presets: [['@babel/preset-env']],
-}

packages/api/build.js

@@ -1,15 +0,0 @@
-const { nodeExternalsPlugin } = require('esbuild-node-externals')
-
-const buildShallow =
-  process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
-
-require('esbuild').build({
-  logLevel: 'info',
-  entryPoints: ['src/index.ts'],
-  bundle: true,
-  sourcemap: true,
-  outdir: 'dist',
-  platform: 'browser',
-  format: 'cjs',
-  plugins: buildShallow ? [nodeExternalsPlugin()] : [],
-})

packages/api/jest.bench.config.js

@@ -1,8 +0,0 @@
-const base = require('./jest.config')
-
-module.exports = {
-  ...base,
-  roots: ['<rootDir>/bench'],
-  testRegex: '(.*.bench.ts)',
-  testTimeout: 3000000,
-}

packages/api/jest.config.js

@@ -1,6 +1,9 @@
-const base = require('../../jest.config.base.js')
+/** @type {import('jest').Config} */
-
 module.exports = {
-  ...base,
   displayName: 'API',
+  transform: { '^.+\\.(t|j)s$': '@swc/jest' },
+  transformIgnorePatterns: [`<rootDir>/node_modules/(?!get-port)`],
+  testTimeout: 60000,
+  setupFiles: ['<rootDir>/../../jest.setup.ts'],
+  setupFilesAfterEnv: ['<rootDir>/jest.setup.ts'],
 }

packages/api/jest.d.ts

@@ -0,0 +1,20 @@
+declare namespace jest {
+  // eslint-disable-next-line
+  interface Matchers<R, T = {}> {
+    toBeModerationResult(
+      expected: ModerationTestSuiteResultFlag[] | undefined,
+      context?: string,
+      stringifiedResult?: string,
+      ignoreCause?: boolean,
+    ): R
+  }
+
+  interface Expect {
+    toBeModerationResult(
+      expected: ModerationTestSuiteResultFlag[] | undefined,
+      context?: string,
+      stringifiedResult?: string,
+      ignoreCause?: boolean,
+    ): void
+  }
+}

packages/api/jest.setup.ts

@@ -0,0 +1,97 @@
+import { ModerationUI } from './src'
+import { ModerationTestSuiteResultFlag } from './tests/util/moderation-behavior'
+
+expect.extend({
+  toBeModerationResult(
+    actual: ModerationUI,
+    expected: ModerationTestSuiteResultFlag[] | undefined,
+    context: string = '',
+    stringifiedResult: string | undefined = undefined,
+    _ignoreCause = false,
+  ) {
+    const fail = (msg: string) => ({
+      pass: false,
+      message: () =>
+        `${msg}.${
+          stringifiedResult ? ` Full result: ${stringifiedResult}` : ''
+        }`,
+    })
+    // let cause = actual.causes?.type as string
+    // if (actual.cause?.type === 'label') {
+    //   cause = `label:${actual.cause.labelDef.id}`
+    // } else if (actual.cause?.type === 'muted') {
+    //   if (actual.cause.source.type === 'list') {
+    //     cause = 'muted-by-list'
+    //   }
+    // } else if (actual.cause?.type === 'blocking') {
+    //   if (actual.cause.source.type === 'list') {
+    //     cause = 'blocking-by-list'
+    //   }
+    // }
+    if (!expected) {
+      // if (!ignoreCause && actual.cause) {
+      //   return fail(`${context} expected to be a no-op, got ${cause}`)
+      // }
+      if (actual.inform) {
+        return fail(`${context} expected to be a no-op, got inform=true`)
+      }
+      if (actual.alert) {
+        return fail(`${context} expected to be a no-op, got alert=true`)
+      }
+      if (actual.blur) {
+        return fail(`${context} expected to be a no-op, got blur=true`)
+      }
+      if (actual.filter) {
+        return fail(`${context} expected to be a no-op, got filter=true`)
+      }
+      if (actual.noOverride) {
+        return fail(`${context} expected to be a no-op, got noOverride=true`)
+      }
+    } else {
+      // if (!ignoreCause && cause !== expected.cause) {
+      //   return fail(`${context} expected to be ${expected.cause}, got ${cause}`)
+      // }
+      const expectedInform = expected.includes('inform')
+      if (!!actual.inform !== expectedInform) {
+        return fail(
+          `${context} expected to be inform=${expectedInform}, got ${
+            actual.inform || false
+          }`,
+        )
+      }
+      const expectedAlert = expected.includes('alert')
+      if (!!actual.alert !== expectedAlert) {
+        return fail(
+          `${context} expected to be alert=${expectedAlert}, got ${
+            actual.alert || false
+          }`,
+        )
+      }
+      const expectedBlur = expected.includes('blur')
+      if (!!actual.blur !== expectedBlur) {
+        return fail(
+          `${context} expected to be blur=${expectedBlur}, got ${
+            actual.blur || false
+          }`,
+        )
+      }
+      const expectedFilter = expected.includes('filter')
+      if (!!actual.filter !== expectedFilter) {
+        return fail(
+          `${context} expected to be filter=${expectedFilter}, got ${
+            actual.filter || false
+          }`,
+        )
+      }
+      const expectedNoOverride = expected.includes('noOverride')
+      if (!!actual.noOverride !== expectedNoOverride) {
+        return fail(
+          `${context} expected to be noOverride=${expectedNoOverride}, got ${
+            actual.noOverride || false
+          }`,
+        )
+      }
+    }
+    return { pass: true, message: () => '' }
+  },
+})

packages/api/package.json

@@ -14,19 +14,12 @@
     "url": "https://github.com/bluesky-social/atproto",
     "directory": "packages/api"
   },
-  "main": "src/index.ts",
+  "main": "dist/index.js",
-  "publishConfig": {
+  "types": "dist/index.d.ts",
-    "main": "dist/index.js",
-    "types": "dist/index.d.ts"
-  },
   "scripts": {
     "codegen": "node ./scripts/generate-code.mjs && lex gen-api ./src/client ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/* ../../lexicons/tools/ozone/*/*",
-    "build": "node ./build.js",
+    "build": "tsc --build tsconfig.build.json",
-    "postbuild": "tsc --build tsconfig.build.json",
+    "test": "jest"
-    "update-main-to-dist": "node ../../update-main-to-dist.js packages/api",
-    "test": "jest",
-    "bench": "jest --config jest.bench.config.js",
-    "bench:profile": "node --inspect-brk ../../node_modules/.bin/jest --config jest.bench.config.js"
   },
   "dependencies": {
     "@atproto/common-web": "workspace:^",
@@ -38,7 +31,7 @@
   },
   "devDependencies": {
     "@atproto/lex-cli": "workspace:^",
-    "@atproto/dev-env": "workspace:^",
+    "get-port": "^6.1.2",
-    "get-port": "^6.1.2"
+    "jest": "^28.1.2"
   }
 }

packages/api/tests/moderation.test.ts

@@ -5,6 +5,7 @@ import {
   interpretLabelValueDefinition,
 } from '../src'
 import './util/moderation-behavior'
+import { ModerationOpts } from '../dist'
 
 describe('Moderation', () => {
   it('Applies self-labels on profiles according to the global preferences', () => {
@@ -30,6 +31,8 @@ describe('Moderation', () => {
             porn: 'hide',
           },
           labelers: [],
+          hiddenPosts: [],
+          mutedWords: [],
         },
       },
     )
@@ -62,6 +65,8 @@ describe('Moderation', () => {
             porn: 'ignore',
           },
           labelers: [],
+          hiddenPosts: [],
+          mutedWords: [],
         },
       },
     )
@@ -96,6 +101,8 @@ describe('Moderation', () => {
             porn: 'hide',
           },
           labelers: [],
+          hiddenPosts: [],
+          mutedWords: [],
         },
       },
     )
@@ -108,7 +115,7 @@ describe('Moderation', () => {
       'contentList',
       'contentView',
       'contentMedia',
-    ]) {
+    ] as const) {
       expect(res1.ui(k)).toBeModerationResult(
         [],
         k,
@@ -143,6 +150,8 @@ describe('Moderation', () => {
               labels: { porn: 'ignore' },
             },
           ],
+          hiddenPosts: [],
+          mutedWords: [],
         },
       },
     )
@@ -155,7 +164,7 @@ describe('Moderation', () => {
       'contentList',
       'contentView',
       'contentMedia',
-    ]) {
+    ] as const) {
       expect(res2.ui(k)).toBeModerationResult(
         [],
         k,
@@ -188,6 +197,8 @@ describe('Moderation', () => {
               labels: {},
             },
           ],
+          hiddenPosts: [],
+          mutedWords: [],
         },
       },
     )
@@ -238,17 +249,19 @@ describe('Moderation', () => {
               labels: {},
             },
           ],
+          hiddenPosts: [],
+          mutedWords: [],
         },
       },
     )
-    expect(res1.ui('contentList').filters[0].label.val).toBe('!hide')
+    expect((res1.ui('contentList').filters[0] as any).label.val).toBe('!hide')
-    expect(res1.ui('contentList').filters[1].label.val).toBe('porn')
+    expect((res1.ui('contentList').filters[1] as any).label.val).toBe('porn')
-    expect(res1.ui('contentList').blurs[0].label.val).toBe('!hide')
+    expect((res1.ui('contentList').blurs[0] as any).label.val).toBe('!hide')
-    expect(res1.ui('contentMedia').blurs[0].label.val).toBe('porn')
+    expect((res1.ui('contentMedia').blurs[0] as any).label.val).toBe('porn')
   })
 
   it('Prioritizes custom label definitions', () => {
-    const modOpts = {
+    const modOpts: ModerationOpts = {
       userDid: 'did:web:alice.test',
       prefs: {
         adultContentEnabled: true,
@@ -259,6 +272,8 @@ describe('Moderation', () => {
             labels: { porn: 'warn' },
           },
         ],
+        hiddenPosts: [],
+        mutedWords: [],
       },
       labelDefs: {
         'did:web:labeler.test': [
@@ -268,6 +283,7 @@ describe('Moderation', () => {
               blurs: 'none',
               severity: 'inform',
               locales: [],
+              defaultSetting: 'warn',
             },
             'did:web:labeler.test',
           ),
@@ -306,7 +322,7 @@ describe('Moderation', () => {
   })
 
   it('Doesnt allow custom behaviors to override imperative labels', () => {
-    const modOpts = {
+    const modOpts: ModerationOpts = {
       userDid: 'did:web:alice.test',
       prefs: {
         adultContentEnabled: true,
@@ -317,6 +333,8 @@ describe('Moderation', () => {
             labels: {},
           },
         ],
+        hiddenPosts: [],
+        mutedWords: [],
       },
       labelDefs: {
         'did:web:labeler.test': [
@@ -326,6 +344,7 @@ describe('Moderation', () => {
               blurs: 'none',
               severity: 'inform',
               locales: [],
+              defaultSetting: 'warn',
             },
             'did:web:labeler.test',
           ),
@@ -369,7 +388,7 @@ describe('Moderation', () => {
   })
 
   it('Ignores invalid label value names', () => {
-    const modOpts = {
+    const modOpts: ModerationOpts = {
       userDid: 'did:web:alice.test',
       prefs: {
         adultContentEnabled: true,
@@ -380,6 +399,8 @@ describe('Moderation', () => {
             labels: { BadLabel: 'hide', 'bad/label': 'hide' },
           },
         ],
+        hiddenPosts: [],
+        mutedWords: [],
       },
       labelDefs: {
         'did:web:labeler.test': [
@@ -389,6 +410,7 @@ describe('Moderation', () => {
               blurs: 'content',
               severity: 'inform',
               locales: [],
+              defaultSetting: 'warn',
             },
             'did:web:labeler.test',
           ),
@@ -398,6 +420,7 @@ describe('Moderation', () => {
               blurs: 'content',
               severity: 'inform',
               locales: [],
+              defaultSetting: 'warn',
             },
             'did:web:labeler.test',
           ),
@@ -443,7 +466,7 @@ describe('Moderation', () => {
   })
 
   it('Custom labels can set the default setting', () => {
-    const modOpts = {
+    const modOpts: ModerationOpts = {
       userDid: 'did:web:alice.test',
       prefs: {
         adultContentEnabled: true,
@@ -454,6 +477,8 @@ describe('Moderation', () => {
             labels: {},
           },
         ],
+        hiddenPosts: [],
+        mutedWords: [],
       },
       labelDefs: {
         'did:web:labeler.test': [
@@ -585,7 +610,7 @@ describe('Moderation', () => {
   })
 
   it('Custom labels can require adult content to be enabled', () => {
-    const modOpts = {
+    const modOpts: ModerationOpts = {
       userDid: 'did:web:alice.test',
       prefs: {
         adultContentEnabled: false,
@@ -598,6 +623,8 @@ describe('Moderation', () => {
             },
           },
         ],
+        hiddenPosts: [],
+        mutedWords: [],
       },
       labelDefs: {
         'did:web:labeler.test': [
@@ -652,7 +679,7 @@ describe('Moderation', () => {
   })
 
   it('Adult content disabled forces the preference to hide', () => {
-    const modOpts = {
+    const modOpts: ModerationOpts = {
       userDid: 'did:web:alice.test',
       prefs: {
         adultContentEnabled: false,
@@ -663,6 +690,8 @@ describe('Moderation', () => {
             labels: {},
           },
         ],
+        hiddenPosts: [],
+        mutedWords: [],
       },
       labelDefs: {},
     }

packages/api/tests/util/moderation-behavior.ts

@@ -65,7 +65,7 @@ expect.extend({
     expected: ModerationTestSuiteResultFlag[] | undefined,
     context: string = '',
     stringifiedResult: string | undefined = undefined,
-    ignoreCause = false,
+    _ignoreCause = false,
   ) {
     const fail = (msg: string) => ({
       pass: false,

packages/api/tsconfig.build.json

@@ -1,4 +1,9 @@
 {
-  "extends": "./tsconfig.json",
+  "extends": "../../tsconfig/isomorphic.json",
-  "exclude": ["**/*.spec.ts", "**/*.test.ts"]
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "noUnusedLocals": false
+  },
+  "include": ["./src"]
 }

packages/api/tsconfig.json

@@ -1,13 +1,7 @@
 {
-  "extends": "../../tsconfig.json",
+  "include": [],
-  "compilerOptions": {
-    "rootDir": "./src",
-    "outDir": "./dist", // Your outDir,
-    "emitDeclarationOnly": true
-  },
-  "include": ["./src"],
   "references": [
-    { "path": "../xrpc/tsconfig.build.json" },
+    { "path": "./tsconfig.build.json" },
-    { "path": "../lex-cli/tsconfig.build.json" }
+    { "path": "./tsconfig.tests.json" }
   ]
 }

packages/api/tsconfig.tests.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig/tests.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "types": ["jest", "./jest.d.ts"],
+    "noEmit": true,
+    "noUnusedLocals": false
+  },
+  "include": ["./tests"]
+}

packages/aws/build.js

@@ -1,14 +0,0 @@
-const { nodeExternalsPlugin } = require('esbuild-node-externals')
-
-const buildShallow =
-  process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
-
-require('esbuild').build({
-  logLevel: 'info',
-  entryPoints: ['src/index.ts'],
-  bundle: true,
-  sourcemap: true,
-  outdir: 'dist',
-  platform: 'node',
-  plugins: buildShallow ? [nodeExternalsPlugin()] : [],
-})

packages/aws/package.json

@@ -13,15 +13,10 @@
     "url": "https://github.com/bluesky-social/atproto",
     "directory": "packages/aws"
   },
-  "main": "src/index.ts",
+  "main": "dist/index.js",
-  "publishConfig": {
+  "types": "dist/index.d.ts",
-    "main": "dist/index.js",
-    "types": "dist/src/index.d.ts"
-  },
   "scripts": {
-    "build": "node ./build.js",
+    "build": "tsc --build tsconfig.build.json"
-    "postbuild": "tsc --build tsconfig.build.json",
-    "update-main-to-dist": "node ../../update-main-to-dist.js packages/aws"
   },
   "dependencies": {
     "@atproto/common": "workspace:^",

packages/aws/tsconfig.build.json

@@ -1,4 +1,8 @@
 {
-  "extends": "./tsconfig.json",
+  "extends": "../../tsconfig/isomorphic.json",
-  "exclude": ["**/*.spec.ts", "**/*.test.ts"]
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "include": ["./src"]
 }

packages/aws/tsconfig.json

@@ -1,8 +1,4 @@
 {
-  "extends": "../../tsconfig.json",
+  "include": [],
-  "compilerOptions": {
+  "references": [{ "path": "./tsconfig.build.json" }]
-    "outDir": "./dist", // Your outDir,
-    "emitDeclarationOnly": true
-  },
-  "include": ["./src", "__tests__/**/**.ts"]
 }

packages/bsky/babel.config.js

@@ -1,3 +0,0 @@
-module.exports = {
-  presets: [['@babel/preset-env']],
-}

packages/bsky/buf.gen.yaml

@@ -3,10 +3,10 @@ plugins:
   - plugin: es
     opt:
       - target=ts
-      - import_extension=.ts
+      - import_extension=
     out: src/proto
   - plugin: connect-es
     opt:
       - target=ts
-      - import_extension=.ts
+      - import_extension=
     out: src/proto

packages/bsky/build.js

@@ -1,19 +0,0 @@
-const { nodeExternalsPlugin } = require('esbuild-node-externals')
-
-const buildShallow =
-  process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
-
-require('esbuild').build({
-  logLevel: 'info',
-  entryPoints: ['src/index.ts'],
-  bundle: true,
-  sourcemap: true,
-  outdir: 'dist',
-  platform: 'node',
-  external: [
-    // Referenced in pg driver, but optional and we don't use it
-    'pg-native',
-    'sharp',
-  ],
-  plugins: buildShallow ? [nodeExternalsPlugin()] : [],
-})

packages/bsky/jest.config.js

@@ -1,6 +1,8 @@
-const base = require('../../jest.config.base.js')
+/** @type {import('jest').Config} */
-
 module.exports = {
-  ...base,
   displayName: 'Bsky App View',
+  transform: { '^.+\\.(t|j)s$': '@swc/jest' },
+  transformIgnorePatterns: [`<rootDir>/node_modules/(?!get-port)`],
+  testTimeout: 60000,
+  setupFiles: ['<rootDir>/../../jest.setup.ts'],
 }

packages/bsky/package.json

@@ -13,17 +13,12 @@
     "url": "https://github.com/bluesky-social/atproto",
     "directory": "packages/bsky"
   },
-  "main": "src/index.ts",
+  "main": "dist/index.js",
-  "publishConfig": {
+  "types": "dist/index.d.ts",
-    "main": "dist/index.js",
-    "types": "dist/index.d.ts"
-  },
   "bin": "dist/bin.js",
   "scripts": {
     "codegen": "lex gen-server ./src/lexicon ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/*",
-    "build": "node ./build.js",
+    "build": "tsc --build tsconfig.build.json",
-    "postbuild": "tsc --build tsconfig.build.json",
-    "update-main-to-dist": "node ../../update-main-to-dist.js packages/bsky",
     "start": "node --enable-source-maps dist/bin.js",
     "test": "../dev-infra/with-test-redis-and-db.sh jest",
     "test:log": "tail -50 test.log | pino-pretty",
@@ -65,7 +60,6 @@
   },
   "devDependencies": {
     "@atproto/api": "workspace:^",
-    "@atproto/dev-env": "workspace:^",
     "@atproto/lex-cli": "workspace:^",
     "@atproto/pds": "workspace:^",
     "@atproto/xrpc": "workspace:^",
@@ -79,6 +73,7 @@
     "@types/pg": "^8.6.6",
     "@types/qs": "^6.9.7",
     "axios": "^0.27.2",
-    "http2-express-bridge": "^1.0.7"
+    "jest": "^28.1.2",
+    "ts-node": "^10.8.2"
   }
 }

packages/bsky/src/data-plane/server/db/db.ts

@@ -169,7 +169,7 @@ const onClientError = (err: Error) => dbLogger.error({ err }, 'db client error')
 // -------
 
 class LeakyTxPlugin implements KyselyPlugin {
-  private txOver: boolean
+  private txOver = false
 
   endTx() {
     this.txOver = true

packages/bsky/src/image/server.ts

@@ -4,7 +4,13 @@ import os from 'os'
 import path from 'path'
 import { Readable } from 'stream'
 import axios, { AxiosError } from 'axios'
-import express, { ErrorRequestHandler, NextFunction } from 'express'
+import express, {
+  Request,
+  Response,
+  Express,
+  ErrorRequestHandler,
+  NextFunction,
+} from 'express'
 import createError, { isHttpError } from 'http-errors'
 import { BlobNotFoundError } from '@atproto/repo'
 import {
@@ -20,7 +26,7 @@ import { retryHttp } from '../util/retry'
 import { ServerConfig } from '../config'
 
 export class ImageProcessingServer {
-  app = express()
+  app: Express = express()
   uriBuilder: ImageUriBuilder
 
   constructor(public cfg: ServerConfig, public cache: BlobCache) {
@@ -29,11 +35,7 @@ export class ImageProcessingServer {
     this.app.use(errorMiddleware)
   }
 
-  async handler(
+  async handler(req: Request, res: Response, next: NextFunction) {
-    req: express.Request,
-    res: express.Response,
-    next: NextFunction,
-  ) {
     try {
       const path = req.path
       const options = ImageUriBuilder.getOptions(path)

packages/bsky/src/index.ts

@@ -29,6 +29,7 @@ export { ServerConfig } from './config'
 export { Database } from './data-plane/server/db'
 export { Redis } from './redis'
 export { AppContext } from './context'
+export { BackgroundQueue } from './data-plane/server/background'
 
 export class BskyAppView {
   public ctx: AppContext

packages/bsky/src/proto/bsky_connect.ts

@@ -1,4 +1,4 @@
-// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension=.ts"
+// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension="
 // @generated from file bsky.proto (package bsky, syntax proto3)
 /* eslint-disable */
 // @ts-nocheck
@@ -160,7 +160,7 @@ import {
   UntakedownRecordResponse,
   UpdateNotificationSeenRequest,
   UpdateNotificationSeenResponse,
-} from './bsky_pb.ts'
+} from './bsky_pb'
 import { MethodKind } from '@bufbuild/protobuf'
 
 /**

packages/bsky/src/proto/bsky_pb.ts

@@ -1,4 +1,4 @@
-// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension=.ts"
+// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension="
 // @generated from file bsky.proto (package bsky, syntax proto3)
 /* eslint-disable */
 // @ts-nocheck

packages/bsky/src/proto/bsync_connect.ts

@@ -1,4 +1,4 @@
-// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension=.ts"
+// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension="
 // @generated from file bsync.proto (package bsync, syntax proto3)
 /* eslint-disable */
 // @ts-nocheck
@@ -10,7 +10,7 @@ import {
   PingResponse,
   ScanMuteOperationsRequest,
   ScanMuteOperationsResponse,
-} from './bsync_pb.ts'
+} from './bsync_pb'
 import { MethodKind } from '@bufbuild/protobuf'
 
 /**

packages/bsky/src/proto/bsync_pb.ts

@@ -1,4 +1,4 @@
-// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension=.ts"
+// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension="
 // @generated from file bsync.proto (package bsync, syntax proto3)
 /* eslint-disable */
 // @ts-nocheck

packages/bsky/src/proto/courier_connect.ts

@@ -1,4 +1,4 @@
-// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension=.ts"
+// @generated by protoc-gen-connect-es v1.3.0 with parameter "target=ts,import_extension="
 // @generated from file courier.proto (package courier, syntax proto3)
 /* eslint-disable */
 // @ts-nocheck
@@ -10,7 +10,7 @@ import {
   PushNotificationsResponse,
   RegisterDeviceTokenRequest,
   RegisterDeviceTokenResponse,
-} from './courier_pb.ts'
+} from './courier_pb'
 import { MethodKind } from '@bufbuild/protobuf'
 
 /**

packages/bsky/src/proto/courier_pb.ts

@@ -1,4 +1,4 @@
-// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension=.ts"
+// @generated by protoc-gen-es v1.6.0 with parameter "target=ts,import_extension="
 // @generated from file courier.proto (package courier, syntax proto3)
 /* eslint-disable */
 // @ts-nocheck

packages/bsky/tests/_util.ts

@@ -8,12 +8,26 @@ import {
   isThreadViewPost,
 } from '../src/lexicon/types/app/bsky/feed/defs'
 import { isViewRecord } from '../src/lexicon/types/app/bsky/embed/record'
+import { AppBskyFeedGetPostThread } from '@atproto/api'
 import {
   LabelerView,
   isLabelerView,
   isLabelerViewDetailed,
 } from '../src/lexicon/types/app/bsky/labeler/defs'
 
+type ThreadViewPost = Extract<
+  AppBskyFeedGetPostThread.OutputSchema['thread'],
+  { post: { uri: string } }
+>
+
+export function assertIsThreadViewPost(
+  value: unknown,
+): asserts value is ThreadViewPost {
+  expect(value).toMatchObject({
+    $type: 'app.bsky.feed.defs#threadViewPost',
+  })
+}
+
 // Swap out identifiers and dates with stable
 // values for the purpose of snapshot testing
 export const forSnapshot = (obj: unknown) => {

packages/bsky/tests/data-plane/duplicate-records.test.ts

@@ -3,7 +3,8 @@ import { cidForCbor, TID } from '@atproto/common'
 import { WriteOpAction } from '@atproto/repo'
 import { TestNetwork } from '@atproto/dev-env'
 import * as lex from '../../src/lexicon/lexicons'
-import { Database } from '../../src'
+
+type Database = TestNetwork['bsky']['db']
 
 describe('duplicate record', () => {
   let network: TestNetwork

packages/bsky/tests/data-plane/indexing.test.ts

@@ -1,7 +1,7 @@
 import { sql } from 'kysely'
 import { CID } from 'multiformats/cid'
 import { cidForCbor, TID } from '@atproto/common'
-import * as pdsRepo from '@atproto/pds/src/repo/prepare'
+import { repoPrepare } from '@atproto/pds'
 import { WriteOpAction } from '@atproto/repo'
 import { AtUri } from '@atproto/syntax'
 import AtpAgent, {
@@ -489,12 +489,12 @@ describe('indexing', () => {
       // const { db: pdsDb, services: pdsServices } = network.pds.ctx
       // Create a good and a bad post record
       const writes = await Promise.all([
-        pdsRepo.prepareCreate({
+        repoPrepare.prepareCreate({
           did: sc.dids.alice,
           collection: ids.AppBskyFeedPost,
           record: { text: 'valid', createdAt: new Date().toISOString() },
         }),
-        pdsRepo.prepareCreate({
+        repoPrepare.prepareCreate({
           did: sc.dids.alice,
           collection: ids.AppBskyFeedPost,
           record: { text: 0 },

packages/bsky/tests/data-plane/subscription/repo.test.ts

@@ -1,13 +1,13 @@
 import AtpAgent from '@atproto/api'
-import { TestNetwork, SeedClient, basicSeed } from '@atproto/dev-env'
-import { CommitData } from '@atproto/repo'
-import { PreparedWrite } from '@atproto/pds/src/repo'
-import * as sequencer from '@atproto/pds/src/sequencer'
 import { cborDecode, cborEncode } from '@atproto/common'
 import { DatabaseSchemaType } from '../../../src/data-plane/server/db/database-schema'
+import { SeedClient, TestNetwork, basicSeed } from '@atproto/dev-env'
+import { PreparedWrite, sequencer } from '@atproto/pds'
+import { CommitData } from '@atproto/repo'
 import { ids } from '../../../src/lexicon/lexicons'
 import { forSnapshot } from '../../_util'
-import { Database } from '../../../src'
+
+type Database = TestNetwork['bsky']['db']
 
 describe('sync', () => {
   let network: TestNetwork

packages/bsky/tests/data-plane/subscription/util.test.ts

@@ -1,10 +1,10 @@
 import { wait } from '@atproto/common'
+import { randomStr } from '@atproto/crypto'
 import {
   ConsecutiveList,
   LatestQueue,
   PartitionedQueue,
 } from '../../../src/data-plane/server/subscription/util'
-import { randomStr } from '../../../../crypto/src'
 
 describe('subscription utils', () => {
   describe('ConsecutiveList', () => {

packages/bsky/tests/feed-generation.test.ts

@@ -11,8 +11,6 @@ import {
   basicSeed,
 } from '@atproto/dev-env'
 import { Handler as SkeletonHandler } from '../src/lexicon/types/app/bsky/feed/getFeedSkeleton'
-import { GeneratorView } from '@atproto/api/src/client/types/app/bsky/feed/defs'
-import { UnknownFeedError } from '@atproto/api/src/client/types/app/bsky/feed/getFeed'
 import { ids } from '../src/lexicon/lexicons'
 import {
   FeedViewPost,
@@ -197,7 +195,7 @@ describe('feed generation', () => {
       return res.data
     }
 
-    const paginatedAll: GeneratorView[] = results(await paginateAll(paginator))
+    const paginatedAll = results(await paginateAll(paginator))
 
     expect(paginatedAll.length).toEqual(5)
     expect(paginatedAll[0].uri).toEqual(feedUriOdd)
@@ -453,7 +451,9 @@ describe('feed generation', () => {
         { feed: feedUriOdd },
         { headers: await network.serviceHeaders(alice, gen.did) },
       )
-      await expect(tryGetFeed).rejects.toThrow(UnknownFeedError)
+      await expect(tryGetFeed).rejects.toMatchObject({
+        error: 'UnknownFeed',
+      })
     })
 
     it('resolves contents of taken-down feed.', async () => {

packages/bsky/tests/image/server.test.ts

@@ -35,7 +35,7 @@ describe('image processing server', () => {
       ImageUriBuilder.getPath({
         preset: 'feed_fullsize',
         did: fileDid,
-        cid: fileCid,
+        cid: fileCid.toString(),
       }),
       { responseType: 'stream' },
     )
@@ -61,7 +61,7 @@ describe('image processing server', () => {
     const path = ImageUriBuilder.getPath({
       preset: 'avatar',
       did: fileDid,
-      cid: fileCid,
+      cid: fileCid.toString(),
     })
     const res1 = await client.get(path, { responseType: 'arraybuffer' })
     expect(res1.headers['x-cache']).toEqual('miss')
@@ -79,7 +79,7 @@ describe('image processing server', () => {
       ImageUriBuilder.getPath({
         preset: 'feed_fullsize',
         did: fileDid,
-        cid: missingCid,
+        cid: missingCid.toString(),
       }),
     )
     expect(res.status).toEqual(404)

packages/bsky/tests/views/block-lists.test.ts

@@ -1,8 +1,6 @@
 import AtpAgent, { AtUri } from '@atproto/api'
 import { TestNetwork, SeedClient, RecordRef, basicSeed } from '@atproto/dev-env'
 import { forSnapshot } from '../_util'
-import { BlockedActorError } from '@atproto/api/src/client/types/app/bsky/feed/getAuthorFeed'
-import { BlockedByActorError } from '@atproto/api/src/client/types/app/bsky/feed/getAuthorFeed'
 
 describe('pds views with blocking from block lists', () => {
   let network: TestNetwork
@@ -165,13 +163,17 @@ describe('pds views with blocking from block lists', () => {
       { actor: carol },
       { headers: await network.serviceHeaders(dan) },
     )
-    await expect(attempt1).rejects.toThrow(BlockedActorError)
+    await expect(attempt1).rejects.toMatchObject({
+      error: 'BlockedActor',
+    })
 
     const attempt2 = agent.api.app.bsky.feed.getAuthorFeed(
       { actor: dan },
       { headers: await network.serviceHeaders(carol) },
     )
-    await expect(attempt2).rejects.toThrow(BlockedByActorError)
+    await expect(attempt2).rejects.toMatchObject({
+      error: 'BlockedByActor',
+    })
   })
 
   it('strips blocked users out of getTimeline', async () => {

packages/bsky/tests/views/blocks.test.ts

@@ -1,14 +1,7 @@
 import assert from 'assert'
 import { TestNetwork, RecordRef, SeedClient, basicSeed } from '@atproto/dev-env'
 import AtpAgent, { AtUri } from '@atproto/api'
-import { BlockedActorError } from '@atproto/api/src/client/types/app/bsky/feed/getAuthorFeed'
+import { assertIsThreadViewPost, forSnapshot } from '../_util'
-import { BlockedByActorError } from '@atproto/api/src/client/types/app/bsky/feed/getAuthorFeed'
-import { isThreadViewPost } from '@atproto/api/src/client/types/app/bsky/feed/defs'
-import {
-  isViewRecord as isEmbedViewRecord,
-  isViewBlocked as isEmbedViewBlocked,
-} from '@atproto/api/src/client/types/app/bsky/embed/record'
-import { forSnapshot } from '../_util'
 
 describe('pds views with blocking', () => {
   let network: TestNetwork
@@ -116,9 +109,9 @@ describe('pds views with blocking', () => {
       { depth: 1, uri: carolReplyToDan.ref.uriStr },
       { headers: await network.serviceHeaders(alice) },
     )
-    if (!isThreadViewPost(thread.thread)) {
+
-      throw new Error('Expected thread view post')
+    assertIsThreadViewPost(thread.thread)
-    }
+
     expect(thread.thread.post.uri).toEqual(carolReplyToDan.ref.uriStr)
     expect(thread.thread.parent).toMatchObject({
       $type: 'app.bsky.feed.defs#blockedPost',
@@ -149,13 +142,17 @@ describe('pds views with blocking', () => {
       { actor: carol },
       { headers: await network.serviceHeaders(dan) },
     )
-    await expect(attempt1).rejects.toThrow(BlockedActorError)
+    await expect(attempt1).rejects.toMatchObject({
+      error: 'BlockedActor',
+    })
 
     const attempt2 = agent.api.app.bsky.feed.getAuthorFeed(
       { actor: dan },
       { headers: await network.serviceHeaders(carol) },
     )
-    await expect(attempt2).rejects.toThrow(BlockedByActorError)
+    await expect(attempt2).rejects.toMatchObject({
+      error: 'BlockedByActor',
+    })
   })
 
   it('strips blocked users out of getTimeline', async () => {
@@ -396,7 +393,8 @@ describe('pds views with blocking', () => {
         { depth: 1, uri: sc.posts[dan][0].ref.uriStr },
         { headers: await network.serviceHeaders(alice) },
       )
-    assert(isThreadViewPost(replyThenBlock.thread))
+    assertIsThreadViewPost(replyThenBlock.thread)
+
     expect(replyThenBlock.thread.replies?.map(getThreadPostUri)).toEqual([
       aliceReplyToDan.ref.uriStr,
     ])
@@ -411,10 +409,12 @@ describe('pds views with blocking', () => {
       { depth: 1, uri: sc.posts[dan][0].ref.uriStr },
       { headers: await network.serviceHeaders(alice) },
     )
-    assert(isThreadViewPost(unblock.thread))
+
-    expect(unblock.thread.replies?.map(getThreadPostUri).sort()).toEqual(
+    assertIsThreadViewPost(unblock.thread)
-      [aliceReplyToDan.ref.uriStr, carolReplyToDan.ref.uriStr].sort(),
+    expect(unblock.thread.replies?.map(getThreadPostUri)).toEqual([
-    )
+      carolReplyToDan.ref.uriStr,
+      aliceReplyToDan.ref.uriStr,
+    ])
 
     // block then reply
     danBlockCarol = await pdsAgent.api.app.bsky.graph.block.create(
@@ -434,7 +434,9 @@ describe('pds views with blocking', () => {
         { depth: 1, uri: sc.posts[dan][0].ref.uriStr },
         { headers: await network.serviceHeaders(alice) },
       )
-    assert(isThreadViewPost(blockThenReply.thread))
+
+    assertIsThreadViewPost(blockThenReply.thread)
+
     expect(replyThenBlock.thread.replies?.map(getThreadPostUri)).toEqual([
       aliceReplyToDan.ref.uriStr,
     ])
@@ -454,8 +456,12 @@ describe('pds views with blocking', () => {
         { depth: 0, uri: sc.posts[dan][1].ref.uriStr },
         { headers: await network.serviceHeaders(alice) },
       )
-    assert(isThreadViewPost(embedThenBlock.thread))
+
-    assert(isEmbedViewBlocked(embedThenBlock.thread.post.embed?.record))
+    assertIsThreadViewPost(embedThenBlock.thread)
+
+    expect(embedThenBlock.thread.post.embed?.record).toMatchObject({
+      $type: 'app.bsky.embed.record#viewBlocked',
+    })
 
     // unblock
     await pdsAgent.api.app.bsky.graph.block.delete(
@@ -467,8 +473,12 @@ describe('pds views with blocking', () => {
       { depth: 0, uri: sc.posts[dan][1].ref.uriStr },
       { headers: await network.serviceHeaders(alice) },
     )
-    assert(isThreadViewPost(unblock.thread))
+
-    assert(isEmbedViewRecord(unblock.thread.post.embed?.record))
+    assertIsThreadViewPost(unblock.thread)
+
+    expect(unblock.thread.post?.embed?.record).toMatchObject({
+      $type: 'app.bsky.embed.record#viewRecord',
+    })
 
     // block then embed
     danBlockCarol = await pdsAgent.api.app.bsky.graph.block.create(
@@ -489,8 +499,11 @@ describe('pds views with blocking', () => {
         { depth: 0, uri: carolEmbedsDan.ref.uriStr },
         { headers: await network.serviceHeaders(alice) },
       )
-    assert(isThreadViewPost(blockThenEmbed.thread))
+    assertIsThreadViewPost(blockThenEmbed.thread)
-    assert(isEmbedViewBlocked(blockThenEmbed.thread.post.embed?.record))
+
+    expect(blockThenEmbed.thread.post.embed?.record).toMatchObject({
+      $type: 'app.bsky.embed.record#viewBlocked',
+    })
 
     // cleanup
     await pdsAgent.api.app.bsky.feed.post.delete(
@@ -517,7 +530,9 @@ describe('pds views with blocking', () => {
       (item) => item.post.uri === embedBlockedUri,
     )
     assert(embedBlockedPost)
-    assert(isEmbedViewBlocked(embedBlockedPost.post.embed?.record))
+    expect(embedBlockedPost.post.embed?.record).toMatchObject({
+      $type: 'app.bsky.embed.record#viewBlocked',
+    })
   })
 
   it('returns a list of blocks', async () => {

packages/bsky/tests/views/thread.test.ts

@@ -1,8 +1,10 @@
 import AtpAgent, { AppBskyFeedGetPostThread } from '@atproto/api'
 import { TestNetwork, SeedClient, basicSeed } from '@atproto/dev-env'
-import { forSnapshot, stripViewerFromThread } from '../_util'
+import {
-import assert from 'assert'
+  assertIsThreadViewPost,
-import { isThreadViewPost } from '@atproto/api/src/client/types/app/bsky/feed/defs'
+  forSnapshot,
+  stripViewerFromThread,
+} from '../_util'
 
 describe('pds thread views', () => {
   let network: TestNetwork
@@ -164,12 +166,12 @@ describe('pds thread views', () => {
         { uri: goodReply1.ref.uriStr },
         { headers: await network.serviceHeaders(alice) },
       )
-    assert(isThreadViewPost(goodReply1Thread.thread))
+    assertIsThreadViewPost(goodReply1Thread.thread)
-    assert(isThreadViewPost(goodReply1Thread.thread.parent))
+    assertIsThreadViewPost(goodReply1Thread.thread.parent)
     expect(goodReply1Thread.thread.parent.post.uri).toEqual(goodRoot.ref.uriStr)
     expect(
       goodReply1Thread.thread.replies?.map((r) => {
-        assert(isThreadViewPost(r))
+        assertIsThreadViewPost(r)
         return r.post.uri
       }),
     ).toEqual([
@@ -182,7 +184,7 @@ describe('pds thread views', () => {
         { uri: badReply.ref.uriStr },
         { headers: await network.serviceHeaders(alice) },
       )
-    assert(isThreadViewPost(badReplyThread.thread))
+    assertIsThreadViewPost(badReplyThread.thread)
     expect(badReplyThread.thread.parent).toBeUndefined() // is not goodReply1
   })
 
@@ -192,7 +194,8 @@ describe('pds thread views', () => {
       { headers: await network.serviceHeaders(bob) },
     )
 
-    assert(isThreadViewPost(thread.thread), 'post does not exist')
+    assertIsThreadViewPost(thread.thread)
+
     const post = thread.thread.post
 
     const postSelfLabels = post.labels

packages/bsky/tsconfig.build.json

@@ -1,4 +1,9 @@
 {
-  "extends": "./tsconfig.json",
+  "extends": "../../tsconfig/node.json",
-  "exclude": ["**/*.spec.ts", "**/*.test.ts"]
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "noUnusedLocals": false
+  },
+  "include": ["./src"]
 }

packages/bsky/tsconfig.json

@@ -1,20 +1,7 @@
 {
-  "extends": "../../tsconfig.json",
+  "include": [],
-  "compilerOptions": {
-    "rootDir": "./src",
-    "outDir": "./dist",
-    "emitDeclarationOnly": true
-  },
-  "module": "nodenext",
-  "include": ["./src", "__tests__/**/**.ts"],
   "references": [
-    { "path": "../api/tsconfig.build.json" },
+    { "path": "./tsconfig.build.json" },
-    { "path": "../common/tsconfig.build.json" },
+    { "path": "./tsconfig.tests.json" }
-    { "path": "../crypto/tsconfig.build.json" },
-    { "path": "../lexicon/tsconfig.build.json" },
-    { "path": "../lex-cli/tsconfig.build.json" },
-    { "path": "../repo/tsconfig.build.json" },
-    { "path": "../syntax/tsconfig.build.json" },
-    { "path": "../xrpc-server/tsconfig.build.json" }
   ]
 }

packages/bsky/tsconfig.tests.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig/tests.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "noUnusedLocals": false
+  },
+  "include": ["./tests"]
+}

packages/bsync/babel.config.js

@@ -1,3 +0,0 @@
-module.exports = {
-  presets: [['@babel/preset-env']],
-}

packages/bsync/build.js

@@ -1,18 +0,0 @@
-const { nodeExternalsPlugin } = require('esbuild-node-externals')
-
-const buildShallow =
-  process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
-
-require('esbuild').build({
-  logLevel: 'info',
-  entryPoints: ['src/index.ts'],
-  bundle: true,
-  sourcemap: true,
-  outdir: 'dist',
-  platform: 'node',
-  external: [
-    // Referenced in pg driver, but optional and we don't use it
-    'pg-native',
-  ],
-  plugins: buildShallow ? [nodeExternalsPlugin()] : [],
-})

packages/bsync/jest.config.js

@@ -1,6 +1,6 @@
-const base = require('../../jest.config.base.js')
+/** @type {import('jest').Config} */
-
 module.exports = {
-  ...base,
   displayName: 'Bsync',
+  transform: { '^.+\\.(t|j)s$': '@swc/jest' },
+  setupFiles: ['<rootDir>/../../jest.setup.ts'],
 }

packages/bsync/package.json

@@ -13,15 +13,10 @@
     "url": "https://github.com/bluesky-social/atproto",
     "directory": "packages/bsync"
   },
-  "main": "src/index.ts",
+  "main": "dist/index.js",
-  "publishConfig": {
+  "types": "dist/index.d.ts",
-    "main": "dist/index.js",
-    "types": "dist/index.d.ts"
-  },
   "scripts": {
-    "build": "node ./build.js",
+    "build": "tsc --build tsconfig.build.json",
-    "postbuild": "tsc --build tsconfig.build.json",
-    "update-main-to-dist": "node ../../update-main-to-dist.js packages/bsync",
     "start": "node --enable-source-maps dist/bin.js",
     "test": "../dev-infra/with-test-db.sh jest",
     "test:log": "tail -50 test.log | pino-pretty",
@@ -45,6 +40,8 @@
     "@bufbuild/buf": "^1.28.1",
     "@bufbuild/protoc-gen-es": "^1.5.0",
     "@connectrpc/protoc-gen-connect-es": "^1.1.4",
-    "@types/pg": "^8.6.6"
+    "@types/pg": "^8.6.6",
+    "jest": "^28.1.2",
+    "ts-node": "^10.8.2"
   }
 }

packages/bsync/src/context.ts

@@ -7,21 +7,25 @@ import { EventEmitter } from 'stream'
 export type AppContextOptions = {
   db: Database
   cfg: ServerConfig
+  shutdown: AbortSignal
 }
 
 export class AppContext {
   db: Database
   cfg: ServerConfig
+  shutdown: AbortSignal
   events: TypedEventEmitter<AppEvents>
 
   constructor(opts: AppContextOptions) {
     this.db = opts.db
     this.cfg = opts.cfg
+    this.shutdown = opts.shutdown
     this.events = new EventEmitter() as TypedEventEmitter<AppEvents>
   }
 
   static async fromConfig(
     cfg: ServerConfig,
+    shutdown: AbortSignal,
     overrides?: Partial<AppContextOptions>,
   ): Promise<AppContext> {
     const db = new Database({
@@ -31,7 +35,7 @@ export class AppContext {
       poolMaxUses: cfg.db.poolMaxUses,
       poolIdleTimeoutMs: cfg.db.poolIdleTimeoutMs,
     })
-    return new AppContext({ db, cfg, ...overrides })
+    return new AppContext({ db, cfg, shutdown, ...overrides })
   }
 }
 

packages/bsync/src/db/index.ts

@@ -34,41 +34,43 @@ export class Database {
     if (instances) {
       this.db = instances.db
       this.pool = instances.pool
-      return
+    } else {
-    }
+      // else create a pool & connect
+      const { schema, url } = opts
+      const pool =
+        opts.pool ??
+        new PgPool({
+          connectionString: url,
+          max: opts.poolSize,
+          maxUses: opts.poolMaxUses,
+          idleTimeoutMillis: opts.poolIdleTimeoutMs,
+        })
 
-    // else create a pool & connect
+      // Select count(*) and other pg bigints as js integer
-    const { schema, url } = opts
+      pgTypes.setTypeParser(pgTypes.builtins.INT8, (n) => parseInt(n, 10))
-    const pool =
+
-      opts.pool ??
+      // Setup schema usage, primarily for test parallelism (each test suite runs in its own pg schema)
-      new PgPool({
+      if (schema && !/^[a-z_]+$/i.test(schema)) {
-        connectionString: url,
+        throw new Error(
-        max: opts.poolSize,
+          `Postgres schema must only contain [A-Za-z_]: ${schema}`,
-        maxUses: opts.poolMaxUses,
+        )
-        idleTimeoutMillis: opts.poolIdleTimeoutMs,
+      }
+
+      pool.on('error', onPoolError)
+      pool.on('connect', (client) => {
+        client.on('error', onClientError)
+        if (schema) {
+          // Shared objects such as extensions will go in the public schema
+          client.query(`SET search_path TO "${schema}",public;`)
+        }
       })
 
-    // Select count(*) and other pg bigints as js integer
+      this.pool = pool
-    pgTypes.setTypeParser(pgTypes.builtins.INT8, (n) => parseInt(n, 10))
+      this.db = new Kysely<DatabaseSchemaType>({
-
+        dialect: new PostgresDialect({ pool }),
-    // Setup schema usage, primarily for test parallelism (each test suite runs in its own pg schema)
+      })
-    if (schema && !/^[a-z_]+$/i.test(schema)) {
-      throw new Error(`Postgres schema must only contain [A-Za-z_]: ${schema}`)
     }
 
-    pool.on('error', onPoolError)
-    pool.on('connect', (client) => {
-      client.on('error', onClientError)
-      if (schema) {
-        // Shared objects such as extensions will go in the public schema
-        client.query(`SET search_path TO "${schema}",public;`)
-      }
-    })
-
-    this.pool = pool
-    this.db = new Kysely<DatabaseSchemaType>({
-      dialect: new PostgresDialect({ pool }),
-    })
     this.migrator = new Migrator({
       db: this.db,
       migrationTableSchema: opts.schema,
@@ -165,7 +167,7 @@ const onClientError = (err: Error) => dbLogger.error({ err }, 'db client error')
 // -------
 
 class LeakyTxPlugin implements KyselyPlugin {
-  private txOver: boolean
+  private txOver = false
 
   endTx() {
     this.txOver = true

packages/bsync/src/index.ts

@@ -19,7 +19,7 @@ export class BsyncService {
   public server: http.Server
   private ac: AbortController
   private terminator: HttpTerminator
-  private dbStatsInterval: NodeJS.Timer
+  private dbStatsInterval?: NodeJS.Timeout
 
   constructor(opts: {
     ctx: AppContext
@@ -36,8 +36,8 @@ export class BsyncService {
     cfg: ServerConfig,
     overrides?: Partial<AppContextOptions>,
   ): Promise<BsyncService> {
-    const ctx = await AppContext.fromConfig(cfg, overrides)
     const ac = new AbortController()
+    const ctx = await AppContext.fromConfig(cfg, ac.signal, overrides)
     const handler = connectNodeAdapter({
       routes: routes(ctx),
       shutdownSignal: ac.signal,
@@ -55,6 +55,9 @@ export class BsyncService {
   }
 
   async start(): Promise<http.Server> {
+    if (this.dbStatsInterval) {
+      throw new Error(`${this.constructor.name} already started`)
+    }
     this.dbStatsInterval = setInterval(() => {
       dbLogger.info(
         {
@@ -77,6 +80,7 @@ export class BsyncService {
     await this.terminator.terminate()
     await this.ctx.db.close()
     clearInterval(this.dbStatsInterval)
+    this.dbStatsInterval = undefined
   }
 
   async setupAppEvents() {

packages/bsync/src/proto/bsync_connect.ts

@@ -10,7 +10,7 @@ import {
   PingResponse,
   ScanMuteOperationsRequest,
   ScanMuteOperationsResponse,
-} from './bsync_pb.ts'
+} from './bsync_pb'
 import { MethodKind } from '@bufbuild/protobuf'
 
 /**

packages/bsync/src/routes/scan-mute-operations.ts

@@ -13,7 +13,10 @@ export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
     const limit = req.limit || 1000
     const cursor = validCursor(req.cursor)
     const nextMuteOpPromise = once(events, createMuteOpChannel, {
-      signal: AbortSignal.timeout(ctx.cfg.service.longPollTimeoutMs),
+      signal: combineSignals(
+        ctx.shutdown,
+        AbortSignal.timeout(ctx.cfg.service.longPollTimeoutMs),
+      ),
     })
     nextMuteOpPromise.catch(() => null) // ensure timeout is always handled
 
@@ -31,6 +34,7 @@ export default (ctx: AppContext): Partial<ServiceImpl<typeof Service>> => ({
       try {
         await nextMuteOpPromise
       } catch (err) {
+        ctx.shutdown.throwIfAborted()
         return new ScanMuteOperationsResponse({
           operations: [],
           cursor: req.cursor,
@@ -67,3 +71,18 @@ const validCursor = (cursor: string): number | null => {
   }
   return int
 }
+
+const combineSignals = (a: AbortSignal, b: AbortSignal) => {
+  const controller = new AbortController()
+  for (const signal of [a, b]) {
+    if (signal.aborted) {
+      controller.abort()
+      return signal
+    }
+    signal.addEventListener('abort', () => controller.abort(signal.reason), {
+      // @ts-ignore https://github.com/DefinitelyTyped/DefinitelyTyped/pull/68625
+      signal: controller.signal,
+    })
+  }
+  return controller.signal
+}

packages/bsync/tsconfig.build.json

@@ -1,4 +1,8 @@
 {
-  "extends": "./tsconfig.json",
+  "extends": "../../tsconfig/node.json",
-  "exclude": ["**/*.spec.ts", "**/*.test.ts"]
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "include": ["./src"]
 }

packages/bsync/tsconfig.json

@@ -1,14 +1,7 @@
 {
-  "extends": "../../tsconfig.json",
+  "include": [],
-  "compilerOptions": {
-    "rootDir": "./src",
-    "outDir": "./dist",
-    "emitDeclarationOnly": true
-  },
-  "module": "nodenext",
-  "include": ["./src", "__tests__/**/**.ts"],
   "references": [
-    { "path": "../common/tsconfig.build.json" },
+    { "path": "./tsconfig.build.json" },
-    { "path": "../common-web/tsconfig.build.json" }
+    { "path": "./tsconfig.tests.json" }
   ]
 }

packages/bsync/tsconfig.tests.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig/tests.json",
+  "compilerOptions": {
+    "rootDir": "."
+  },
+  "include": ["./tests"]
+}

packages/common-web/babel.config.js

@@ -1 +0,0 @@
-module.exports = require('../../babel.config.js')

packages/common-web/build.js

@@ -1,15 +0,0 @@
-const { nodeExternalsPlugin } = require('esbuild-node-externals')
-
-const buildShallow =
-  process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
-
-require('esbuild').build({
-  logLevel: 'info',
-  entryPoints: ['src/index.ts'],
-  bundle: true,
-  sourcemap: true,
-  outdir: 'dist',
-  platform: 'browser',
-  format: 'cjs',
-  plugins: buildShallow ? [nodeExternalsPlugin()] : [],
-})

packages/common-web/jest.config.js

@@ -1,6 +1,6 @@
-const base = require('../../jest.config.base.js')
+/** @type {import('jest').Config} */
-
 module.exports = {
-  ...base,
   displayName: 'Common Web',
+  transform: { '^.+\\.(t|j)s$': '@swc/jest' },
+  setupFiles: ['<rootDir>/../../jest.setup.ts'],
 }

packages/common-web/package.json

@@ -12,21 +12,19 @@
     "url": "https://github.com/bluesky-social/atproto",
     "directory": "packages/common-web"
   },
-  "main": "src/index.ts",
+  "main": "dist/index.js",
-  "publishConfig": {
+  "types": "dist/index.d.ts",
-    "main": "dist/index.js",
-    "types": "dist/index.d.ts"
-  },
   "scripts": {
     "test": "jest",
-    "build": "node ./build.js",
+    "build": "tsc --build tsconfig.build.json"
-    "postbuild": "tsc --build tsconfig.build.json",
-    "update-main-to-dist": "node ../../update-main-to-dist.js packages/common-web"
   },
   "dependencies": {
     "graphemer": "^1.4.0",
     "multiformats": "^9.9.0",
     "uint8arrays": "3.0.0",
     "zod": "^3.21.4"
+  },
+  "devDependencies": {
+    "jest": "^28.1.2"
   }
 }

packages/common-web/tsconfig.build.json

@@ -1,4 +1,8 @@
 {
-  "extends": "./tsconfig.json",
+  "extends": "../../tsconfig/isomorphic.json",
-  "exclude": ["**/*.spec.ts", "**/*.test.ts"]
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "include": ["./src"]
 }

packages/common-web/tsconfig.json

@@ -1,9 +1,7 @@
 {
-  "extends": "../../tsconfig.json",
+  "include": [],
-  "compilerOptions": {
+  "references": [
-    "rootDir": "./src",
+    { "path": "./tsconfig.build.json" },
-    "outDir": "./dist", // Your outDir,
+    { "path": "./tsconfig.tests.json" }
-    "emitDeclarationOnly": true
+  ]
-  },
-  "include": ["./src", "__tests__/**/**.ts"]
 }

packages/common-web/tsconfig.tests.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig/tests.json",
+  "compilerOptions": {
+    "rootDir": "."
+  },
+  "include": ["./tests"]
+}

packages/common/babel.config.js

@@ -1 +0,0 @@
-module.exports = require('../../babel.config.js')

packages/common/build.js

@@ -1,14 +0,0 @@
-const { nodeExternalsPlugin } = require('esbuild-node-externals')
-
-const buildShallow =
-  process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
-
-require('esbuild').build({
-  logLevel: 'info',
-  entryPoints: ['src/index.ts'],
-  bundle: true,
-  sourcemap: true,
-  outdir: 'dist',
-  platform: 'node',
-  plugins: buildShallow ? [nodeExternalsPlugin()] : [],
-})

packages/common/jest.config.js

@@ -1,6 +1,6 @@
-const base = require('../../jest.config.base.js')
+/** @type {import('jest').Config} */
-
 module.exports = {
-  ...base,
   displayName: 'Common',
+  transform: { '^.+\\.(t|j)s$': '@swc/jest' },
+  setupFiles: ['<rootDir>/../../jest.setup.ts'],
 }

packages/common/package.json

@@ -12,16 +12,11 @@
     "url": "https://github.com/bluesky-social/atproto",
     "directory": "packages/common"
   },
-  "main": "src/index.ts",
+  "main": "dist/index.js",
-  "publishConfig": {
+  "types": "dist/index.d.ts",
-    "main": "dist/index.js",
-    "types": "dist/index.d.ts"
-  },
   "scripts": {
     "test": "jest",
-    "build": "node ./build.js",
+    "build": "tsc --build tsconfig.build.json"
-    "postbuild": "tsc --build tsconfig.build.json",
-    "update-main-to-dist": "node ../../update-main-to-dist.js packages/common"
   },
   "dependencies": {
     "@atproto/common-web": "workspace:^",
@@ -30,5 +25,10 @@
     "iso-datestring-validator": "^2.2.2",
     "multiformats": "^9.9.0",
     "pino": "^8.15.0"
+  },
+  "devDependencies": {
+    "jest": "^28.1.2",
+    "typescript": "^5.3.3",
+    "uint8arrays": "3.0.0"
   }
 }

packages/common/src/streams.ts

@@ -59,8 +59,9 @@ export class MaxSizeChecker extends Transform {
   _transform(chunk: Uint8Array, _enc: BufferEncoding, cb: TransformCallback) {
     this.totalSize += chunk.length
     if (this.totalSize > this.maxSize) {
-      return this.destroy(this.createError())
+      cb(this.createError())
+    } else {
+      cb(null, chunk)
     }
-    return cb(null, chunk)
   }
 }

packages/common/tsconfig.build.json

@@ -1,4 +1,8 @@
 {
-  "extends": "./tsconfig.json",
+  "extends": "../../tsconfig/node.json",
-  "exclude": ["**/*.spec.ts", "**/*.test.ts"]
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "include": ["./src"]
 }

packages/common/tsconfig.json

@@ -1,10 +1,7 @@
 {
-  "extends": "../../tsconfig.json",
+  "include": [],
-  "compilerOptions": {
+  "references": [
-    "rootDir": "./src",
+    { "path": "./tsconfig.build.json" },
-    "outDir": "./dist", // Your outDir,
+    { "path": "./tsconfig.tests.json" }
-    "emitDeclarationOnly": true
+  ]
-  },
-  "include": ["./src", "__tests__/**/**.ts"],
-  "references": [{ "path": "../common-web/tsconfig.build.json" }]
 }

packages/common/tsconfig.tests.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig/tests.json",
+  "compilerOptions": {
+    "rootDir": "."
+  },
+  "include": ["./tests"]
+}

packages/crypto/build.js

@@ -1,14 +0,0 @@
-const { nodeExternalsPlugin } = require('esbuild-node-externals')
-
-const buildShallow =
-  process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
-
-require('esbuild').build({
-  logLevel: 'info',
-  entryPoints: ['src/index.ts'],
-  bundle: true,
-  sourcemap: true,
-  outdir: 'dist',
-  platform: 'node',
-  plugins: buildShallow ? [nodeExternalsPlugin()] : [],
-})

packages/crypto/jest.config.js

@@ -1,6 +1,6 @@
-const base = require('../../jest.config.base.js')
+/** @type {import('jest').Config} */
-
 module.exports = {
-  ...base,
   displayName: 'Crypto',
+  transform: { '^.+\\.(t|j)s$': '@swc/jest' },
+  setupFiles: ['<rootDir>/../../jest.setup.ts'],
 }

packages/crypto/package.json

@@ -13,16 +13,11 @@
     "url": "https://github.com/bluesky-social/atproto",
     "directory": "packages/crypto"
   },
-  "main": "src/index.ts",
+  "main": "dist/index.js",
-  "publishConfig": {
+  "types": "dist/index.d.ts",
-    "main": "dist/index.js",
-    "types": "dist/index.d.ts"
-  },
   "scripts": {
     "test": "jest ",
-    "build": "node ./build.js",
+    "build": "tsc --build tsconfig.build.json"
-    "postbuild": "tsc --build tsconfig.build.json",
-    "update-main-to-dist": "node ../../update-main-to-dist.js packages/crypto"
   },
   "dependencies": {
     "@noble/curves": "^1.1.0",
@@ -30,6 +25,7 @@
     "uint8arrays": "3.0.0"
   },
   "devDependencies": {
-    "@atproto/common": "workspace:^"
+    "@atproto/common": "workspace:^",
+    "jest": "^28.1.2"
   }
 }

packages/crypto/src/did.ts

@@ -1,13 +1,8 @@
 import * as uint8arrays from 'uint8arrays'
-import * as p256 from './p256/encoding'
+
-import * as secp from './secp256k1/encoding'
+import { BASE58_MULTIBASE_PREFIX, DID_KEY_PREFIX } from './const'
 import plugins from './plugins'
-import {
+import { extractMultikey, extractPrefixedBytes, hasPrefix } from './utils'
-  BASE58_MULTIBASE_PREFIX,
-  DID_KEY_PREFIX,
-  P256_JWT_ALG,
-  SECP256K1_JWT_ALG,
-} from './const'
 
 export type ParsedMultikey = {
   jwtAlg: string
@@ -15,23 +10,14 @@ export type ParsedMultikey = {
 }
 
 export const parseMultikey = (multikey: string): ParsedMultikey => {
-  if (!multikey.startsWith(BASE58_MULTIBASE_PREFIX)) {
+  const prefixedBytes = extractPrefixedBytes(multikey)
-    throw new Error(`Incorrect prefix for multikey: ${multikey}`)
-  }
-  const prefixedBytes = uint8arrays.fromString(
-    multikey.slice(BASE58_MULTIBASE_PREFIX.length),
-    'base58btc',
-  )
   const plugin = plugins.find((p) => hasPrefix(prefixedBytes, p.prefix))
   if (!plugin) {
     throw new Error('Unsupported key type')
   }
-  let keyBytes = prefixedBytes.slice(plugin.prefix.length)
+  const keyBytes = plugin.decompressPubkey(
-  if (plugin.jwtAlg === P256_JWT_ALG) {
+    prefixedBytes.slice(plugin.prefix.length),
-    keyBytes = p256.decompressPubkey(keyBytes)
+  )
-  } else if (plugin.jwtAlg === SECP256K1_JWT_ALG) {
-    keyBytes = secp.decompressPubkey(keyBytes)
-  }
   return {
     jwtAlg: plugin.jwtAlg,
     keyBytes,
@@ -46,28 +32,20 @@ export const formatMultikey = (
   if (!plugin) {
     throw new Error('Unsupported key type')
   }
-  if (jwtAlg === P256_JWT_ALG) {
+  const prefixedBytes = uint8arrays.concat([
-    keyBytes = p256.compressPubkey(keyBytes)
+    plugin.prefix,
-  } else if (jwtAlg === SECP256K1_JWT_ALG) {
+    plugin.compressPubkey(keyBytes),
-    keyBytes = secp.compressPubkey(keyBytes)
+  ])
-  }
-  const prefixedBytes = uint8arrays.concat([plugin.prefix, keyBytes])
   return (
     BASE58_MULTIBASE_PREFIX + uint8arrays.toString(prefixedBytes, 'base58btc')
   )
 }
 
 export const parseDidKey = (did: string): ParsedMultikey => {
-  if (!did.startsWith(DID_KEY_PREFIX)) {
+  const multikey = extractMultikey(did)
-    throw new Error(`Incorrect prefix for did:key: ${did}`)
+  return parseMultikey(multikey)
-  }
-  return parseMultikey(did.slice(DID_KEY_PREFIX.length))
 }
 
 export const formatDidKey = (jwtAlg: string, keyBytes: Uint8Array): string => {
   return DID_KEY_PREFIX + formatMultikey(jwtAlg, keyBytes)
 }
-
-const hasPrefix = (bytes: Uint8Array, prefix: Uint8Array): boolean => {
-  return uint8arrays.equals(prefix, bytes.subarray(0, prefix.byteLength))
-}

packages/crypto/src/p256/keypair.ts

@@ -1,7 +1,11 @@
 import { p256 } from '@noble/curves/p256'
 import { sha256 } from '@noble/hashes/sha256'
-import * as uint8arrays from 'uint8arrays'
+import { SupportedEncodings } from 'uint8arrays/to-string'
-import { SupportedEncodings } from 'uint8arrays/util/bases'
+import {
+  fromString as ui8FromString,
+  toString as ui8ToString,
+} from 'uint8arrays'
+
 import * as did from '../did'
 import { P256_JWT_ALG } from '../const'
 import { Keypair } from '../types'
@@ -32,9 +36,7 @@ export class P256Keypair implements Keypair {
   ): Promise<P256Keypair> {
     const { exportable = false } = opts || {}
     const privKeyBytes =
-      typeof privKey === 'string'
+      typeof privKey === 'string' ? ui8FromString(privKey, 'hex') : privKey
-        ? uint8arrays.fromString(privKey, 'hex')
-        : privKey
     return new P256Keypair(privKeyBytes, exportable)
   }
 
@@ -43,7 +45,7 @@ export class P256Keypair implements Keypair {
   }
 
   publicKeyStr(encoding: SupportedEncodings = 'base64pad'): string {
-    return uint8arrays.toString(this.publicKey, encoding)
+    return ui8ToString(this.publicKey, encoding)
   }
 
   did(): string {

packages/crypto/src/p256/operations.ts

@@ -1,9 +1,10 @@
 import { p256 } from '@noble/curves/p256'
 import { sha256 } from '@noble/hashes/sha256'
-import * as ui8 from 'uint8arrays'
+import { equals as ui8equals } from 'uint8arrays'
-import { P256_JWT_ALG } from '../const'
+
-import { parseDidKey } from '../did'
+import { P256_DID_PREFIX } from '../const'
 import { VerifyOptions } from '../types'
+import { extractMultikey, extractPrefixedBytes, hasPrefix } from '../utils'
 
 export const verifyDidSig = async (
   did: string,
@@ -11,10 +12,11 @@ export const verifyDidSig = async (
   sig: Uint8Array,
   opts?: VerifyOptions,
 ): Promise<boolean> => {
-  const { jwtAlg, keyBytes } = parseDidKey(did)
+  const prefixedBytes = extractPrefixedBytes(extractMultikey(did))
-  if (jwtAlg !== P256_JWT_ALG) {
+  if (!hasPrefix(prefixedBytes, P256_DID_PREFIX)) {
     throw new Error(`Not a P-256 did:key: ${did}`)
   }
+  const keyBytes = prefixedBytes.slice(P256_DID_PREFIX.length)
   return verifySig(keyBytes, data, sig, opts)
 }
 
@@ -39,7 +41,7 @@ export const verifySig = async (
 export const isCompactFormat = (sig: Uint8Array) => {
   try {
     const parsed = p256.Signature.fromCompact(sig)
-    return ui8.equals(parsed.toCompactRawBytes(), sig)
+    return ui8equals(parsed.toCompactRawBytes(), sig)
   } catch {
     return false
   }

packages/crypto/src/p256/plugin.ts

@@ -1,11 +1,16 @@
-import * as operations from './operations'
+import { verifyDidSig } from './operations'
+import { compressPubkey, decompressPubkey } from './encoding'
+
 import { DidKeyPlugin } from '../types'
 import { P256_DID_PREFIX, P256_JWT_ALG } from '../const'
 
 export const p256Plugin: DidKeyPlugin = {
   prefix: P256_DID_PREFIX,
   jwtAlg: P256_JWT_ALG,
-  verifySignature: operations.verifyDidSig,
+  verifySignature: verifyDidSig,
+
+  compressPubkey,
+  decompressPubkey,
 }
 
 export default p256Plugin

packages/crypto/src/secp256k1/keypair.ts

@@ -1,9 +1,13 @@
 import { secp256k1 as k256 } from '@noble/curves/secp256k1'
 import { sha256 } from '@noble/hashes/sha256'
-import * as uint8arrays from 'uint8arrays'
+import {
-import { SupportedEncodings } from 'uint8arrays/util/bases'
+  fromString as ui8FromString,
-import * as did from '../did'
+  toString as ui8ToString,
+} from 'uint8arrays'
+import { SupportedEncodings } from 'uint8arrays/to-string'
+
 import { SECP256K1_JWT_ALG } from '../const'
+import * as did from '../did'
 import { Keypair } from '../types'
 
 export type Secp256k1KeypairOptions = {
@@ -32,9 +36,7 @@ export class Secp256k1Keypair implements Keypair {
   ): Promise<Secp256k1Keypair> {
     const { exportable = false } = opts || {}
     const privKeyBytes =
-      typeof privKey === 'string'
+      typeof privKey === 'string' ? ui8FromString(privKey, 'hex') : privKey
-        ? uint8arrays.fromString(privKey, 'hex')
-        : privKey
     return new Secp256k1Keypair(privKeyBytes, exportable)
   }
 
@@ -43,7 +45,7 @@ export class Secp256k1Keypair implements Keypair {
   }
 
   publicKeyStr(encoding: SupportedEncodings = 'base64pad'): string {
-    return uint8arrays.toString(this.publicKey, encoding)
+    return ui8ToString(this.publicKey, encoding)
   }
 
   did(): string {

packages/crypto/src/secp256k1/operations.ts

@@ -1,9 +1,10 @@
 import { secp256k1 as k256 } from '@noble/curves/secp256k1'
 import { sha256 } from '@noble/hashes/sha256'
 import * as ui8 from 'uint8arrays'
-import { SECP256K1_JWT_ALG } from '../const'
+
-import { parseDidKey } from '../did'
+import { SECP256K1_DID_PREFIX } from '../const'
 import { VerifyOptions } from '../types'
+import { extractMultikey, extractPrefixedBytes, hasPrefix } from '../utils'
 
 export const verifyDidSig = async (
   did: string,
@@ -11,10 +12,11 @@ export const verifyDidSig = async (
   sig: Uint8Array,
   opts?: VerifyOptions,
 ): Promise<boolean> => {
-  const { jwtAlg, keyBytes } = parseDidKey(did)
+  const prefixedBytes = extractPrefixedBytes(extractMultikey(did))
-  if (jwtAlg !== SECP256K1_JWT_ALG) {
+  if (!hasPrefix(prefixedBytes, SECP256K1_DID_PREFIX)) {
     throw new Error(`Not a secp256k1 did:key: ${did}`)
   }
+  const keyBytes = prefixedBytes.slice(SECP256K1_DID_PREFIX.length)
   return verifySig(keyBytes, data, sig, opts)
 }
 

packages/crypto/src/secp256k1/plugin.ts

@@ -1,11 +1,16 @@
-import * as operations from './operations'
+import { verifyDidSig } from './operations'
+import { compressPubkey, decompressPubkey } from './encoding'
+
 import { DidKeyPlugin } from '../types'
 import { SECP256K1_DID_PREFIX, SECP256K1_JWT_ALG } from '../const'
 
 export const secp256k1Plugin: DidKeyPlugin = {
   prefix: SECP256K1_DID_PREFIX,
   jwtAlg: SECP256K1_JWT_ALG,
-  verifySignature: operations.verifyDidSig,
+  verifySignature: verifyDidSig,
+
+  compressPubkey,
+  decompressPubkey,
 }
 
 export default secp256k1Plugin

packages/crypto/src/types.ts

@@ -22,6 +22,9 @@ export type DidKeyPlugin = {
     data: Uint8Array,
     opts?: VerifyOptions,
   ) => Promise<boolean>
+
+  compressPubkey: (uncompressed: Uint8Array) => Uint8Array
+  decompressPubkey: (compressed: Uint8Array) => Uint8Array
 }
 
 export type VerifyOptions = {

packages/crypto/src/utils.ts

@@ -0,0 +1,23 @@
+import * as uint8arrays from 'uint8arrays'
+import { BASE58_MULTIBASE_PREFIX, DID_KEY_PREFIX } from './const'
+
+export const extractMultikey = (did: string): string => {
+  if (!did.startsWith(DID_KEY_PREFIX)) {
+    throw new Error(`Incorrect prefix for did:key: ${did}`)
+  }
+  return did.slice(DID_KEY_PREFIX.length)
+}
+
+export const extractPrefixedBytes = (multikey: string): Uint8Array => {
+  if (!multikey.startsWith(BASE58_MULTIBASE_PREFIX)) {
+    throw new Error(`Incorrect prefix for multikey: ${multikey}`)
+  }
+  return uint8arrays.fromString(
+    multikey.slice(BASE58_MULTIBASE_PREFIX.length),
+    'base58btc',
+  )
+}
+
+export const hasPrefix = (bytes: Uint8Array, prefix: Uint8Array): boolean => {
+  return uint8arrays.equals(prefix, bytes.subarray(0, prefix.byteLength))
+}

packages/crypto/tests/signatures.test.ts

@@ -137,6 +137,7 @@ describe('signatures', () => {
   })
 })
 
+// @ts-expect-error
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 async function generateTestVectors(): Promise<TestVector[]> {
   const p256Key = await EcdsaKeypair.create({ exportable: true })

packages/crypto/tsconfig.build.json

@@ -1,4 +1,8 @@
 {
-  "extends": "./tsconfig.json",
+  "extends": "../../tsconfig/isomorphic.json",
-  "exclude": ["**/*.spec.ts", "**/*.test.ts"]
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "include": ["./src"]
 }

packages/crypto/tsconfig.json

@@ -1,9 +1,7 @@
 {
-  "extends": "../../tsconfig.json",
+  "include": [],
-  "compilerOptions": {
+  "references": [
-    "rootDir": "./src",
+    { "path": "./tsconfig.build.json" },
-    "outDir": "./dist", // Your outDir,
+    { "path": "./tsconfig.tests.json" }
-    "emitDeclarationOnly": true
+  ]
-  },
-  "include": ["./src", "__tests__/**/**.ts"]
 }

packages/crypto/tsconfig.tests.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig/tests.json",
+  "compilerOptions": {
+    "rootDir": "."
+  },
+  "include": ["./tests"]
+}

packages/dev-env/build.js

@@ -1,27 +0,0 @@
-const { nodeExternalsPlugin } = require('esbuild-node-externals')
-const hbsPlugin = require('esbuild-plugin-handlebars')
-
-const buildShallow =
-  process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
-
-require('esbuild').build({
-  logLevel: 'info',
-  entryPoints: ['src/index.ts', 'src/bin.ts'],
-  bundle: true,
-  sourcemap: true,
-  outdir: 'dist',
-  platform: 'node',
-  external: [
-    'better-sqlite3',
-    // Referenced in pg driver, but optional and we don't use it
-    'pg-native',
-    'sharp',
-  ],
-  plugins: [].concat(buildShallow ? [nodeExternalsPlugin()] : []).concat([
-    hbsPlugin({
-      filter: /\.(hbs)$/,
-      additionalHelpers: {},
-      precompileOptions: {},
-    }),
-  ]),
-})

packages/dev-env/package.json

@@ -12,16 +12,11 @@
     "url": "https://github.com/bluesky-social/atproto",
     "directory": "packages/dev-env"
   },
-  "main": "src/index.ts",
+  "main": "dist/index.js",
-  "publishConfig": {
+  "types": "dist/index.d.ts",
-    "main": "dist/index.js",
-    "types": "dist/index.d.ts"
-  },
   "bin": "dist/bin.js",
   "scripts": {
-    "build": "node ./build.js",
+    "build": "tsc --build tsconfig.build.json",
-    "postbuild": "tsc --build tsconfig.build.json",
-    "update-main-to-dist": "node ../../update-main-to-dist.js packages/dev-env",
     "start": "../dev-infra/with-test-redis-and-db.sh node dist/bin.js"
   },
   "dependencies": {
@@ -39,15 +34,10 @@
     "@did-plc/lib": "^0.0.1",
     "@did-plc/server": "^0.0.1",
     "axios": "^0.27.2",
-    "better-sqlite3": "^9.4.0",
     "dotenv": "^16.0.3",
     "express": "^4.18.2",
     "get-port": "^6.1.2",
     "multiformats": "^9.9.0",
-    "sharp": "^0.32.6",
     "uint8arrays": "3.0.0"
-  },
-  "devDependencies": {
-    "ts-node": "^10.8.1"
   }
 }

packages/dev-env/src/bsky.ts

@@ -3,10 +3,12 @@ import * as ui8 from 'uint8arrays'
 import * as bsky from '@atproto/bsky'
 import { AtpAgent } from '@atproto/api'
 import { Secp256k1Keypair } from '@atproto/crypto'
+import { BackgroundQueue } from '@atproto/bsky'
 import { Client as PlcClient } from '@did-plc/lib'
 import { BskyConfig } from './types'
 import { ADMIN_PASSWORD, EXAMPLE_LABELER } from './const'
-import { BackgroundQueue } from '@atproto/bsky/src/data-plane/server/background'
+
+export * from '@atproto/bsky'
 
 export class TestBsky {
   constructor(

packages/dev-env/src/feed-gen.ts

@@ -1,12 +1,11 @@
-import http from 'http'
+import { Secp256k1Keypair } from '@atproto/crypto'
+import { SkeletonHandler, createLexiconServer } from '@atproto/pds'
+import { InvalidRequestError } from '@atproto/xrpc-server'
+import * as plc from '@did-plc/lib'
 import events from 'events'
 import express from 'express'
 import getPort from 'get-port'
-import * as plc from '@did-plc/lib'
+import http from 'http'
-import { Secp256k1Keypair } from '@atproto/crypto'
-import { Handler as SkeletonHandler } from '@atproto/pds/src/lexicon/types/app/bsky/feed/getFeedSkeleton'
-import { createServer } from '@atproto/pds/src/lexicon'
-import { InvalidRequestError } from '@atproto/xrpc-server'
 
 export class TestFeedGen {
   destroyed = false
@@ -24,7 +23,7 @@ export class TestFeedGen {
     const port = await getPort()
     const did = await createFgDid(plcUrl, port)
     const app = express()
-    const lexServer = createServer()
+    const lexServer = createLexiconServer()
 
     lexServer.app.bsky.feed.getFeedSkeleton(async (args) => {
       const handler = feeds[args.params.feed]

packages/dev-env/src/index.ts

@@ -2,6 +2,7 @@ export * from './bsky'
 export * from './bsync'
 export * from './network'
 export * from './network-no-appview'
+export * from './ozone'
 export * from './pds'
 export * from './plc'
 export * from './ozone'