close CLOUD-137 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Introduced advanced workspace-scoped search and aggregation capabilities with support for complex queries, highlights, and pagination. - Added pluggable search providers: Elasticsearch and Manticoresearch. - New GraphQL queries, schema types, and resolver support for search and aggregation. - Enhanced configuration options for search providers in self-hosted and cloud deployments. - Added Docker Compose services and environment variables for Elasticsearch and Manticoresearch. - Integrated indexer service into deployment and CI workflows. - **Bug Fixes** - Improved error handling with new user-friendly error messages for search provider and indexer issues. - **Documentation** - Updated configuration examples and environment variable references for indexer and search providers. - **Tests** - Added extensive end-to-end and provider-specific tests covering indexing, searching, aggregation, deletion, and error cases. - Included snapshot tests and test fixtures for search providers. - **Chores** - Updated deployment scripts, Helm charts, and Kubernetes manifests to include indexer-related environment variables and secrets. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
179 lines
6.3 KiB
JavaScript
179 lines
6.3 KiB
JavaScript
import { execSync } from 'node:child_process';
|
|
|
|
const {
|
|
APP_VERSION,
|
|
BUILD_TYPE,
|
|
DEPLOY_HOST,
|
|
CANARY_DEPLOY_HOST,
|
|
GIT_SHORT_HASH,
|
|
DATABASE_URL,
|
|
DATABASE_USERNAME,
|
|
DATABASE_PASSWORD,
|
|
DATABASE_NAME,
|
|
GCLOUD_CONNECTION_NAME,
|
|
CLOUD_SQL_IAM_ACCOUNT,
|
|
APP_IAM_ACCOUNT,
|
|
REDIS_SERVER_HOST,
|
|
REDIS_SERVER_PASSWORD,
|
|
STATIC_IP_NAME,
|
|
AFFINE_INDEXER_SEARCH_PROVIDER,
|
|
AFFINE_INDEXER_SEARCH_ENDPOINT,
|
|
AFFINE_INDEXER_SEARCH_USERNAME,
|
|
AFFINE_INDEXER_SEARCH_PASSWORD,
|
|
} = process.env;
|
|
|
|
const buildType = BUILD_TYPE || 'canary';
|
|
|
|
const isProduction = buildType === 'stable';
|
|
const isBeta = buildType === 'beta';
|
|
const isInternal = buildType === 'internal';
|
|
|
|
const replicaConfig = {
|
|
stable: {
|
|
web: 3,
|
|
graphql: Number(process.env.PRODUCTION_GRAPHQL_REPLICA) || 3,
|
|
sync: Number(process.env.PRODUCTION_SYNC_REPLICA) || 3,
|
|
renderer: Number(process.env.PRODUCTION_RENDERER_REPLICA) || 3,
|
|
doc: Number(process.env.PRODUCTION_DOC_REPLICA) || 3,
|
|
},
|
|
beta: {
|
|
web: 2,
|
|
graphql: Number(process.env.BETA_GRAPHQL_REPLICA) || 2,
|
|
sync: Number(process.env.BETA_SYNC_REPLICA) || 2,
|
|
renderer: Number(process.env.BETA_RENDERER_REPLICA) || 2,
|
|
doc: Number(process.env.BETA_DOC_REPLICA) || 2,
|
|
},
|
|
canary: {
|
|
web: 2,
|
|
graphql: 2,
|
|
sync: 2,
|
|
renderer: 2,
|
|
doc: 2,
|
|
},
|
|
};
|
|
|
|
const cpuConfig = {
|
|
beta: {
|
|
web: '300m',
|
|
graphql: '1',
|
|
sync: '1',
|
|
doc: '1',
|
|
renderer: '300m',
|
|
},
|
|
canary: {
|
|
web: '300m',
|
|
graphql: '1',
|
|
sync: '1',
|
|
doc: '1',
|
|
renderer: '300m',
|
|
},
|
|
};
|
|
|
|
const createHelmCommand = ({ isDryRun }) => {
|
|
const flag = isDryRun ? '--dry-run' : '--atomic';
|
|
const imageTag = `${buildType}-${GIT_SHORT_HASH}`;
|
|
const redisAndPostgres =
|
|
isProduction || isBeta || isInternal
|
|
? [
|
|
`--set cloud-sql-proxy.enabled=true`,
|
|
`--set-string cloud-sql-proxy.database.connectionName="${GCLOUD_CONNECTION_NAME}"`,
|
|
`--set-string global.database.host=${DATABASE_URL}`,
|
|
`--set-string global.database.user=${DATABASE_USERNAME}`,
|
|
`--set-string global.database.password=${DATABASE_PASSWORD}`,
|
|
`--set-string global.database.name=${DATABASE_NAME}`,
|
|
`--set-string global.redis.host="${REDIS_SERVER_HOST}"`,
|
|
`--set-string global.redis.password="${REDIS_SERVER_PASSWORD}"`,
|
|
]
|
|
: [];
|
|
const indexerOptions = [
|
|
`--set-string global.indexer.provider="${AFFINE_INDEXER_SEARCH_PROVIDER}"`,
|
|
`--set-string global.indexer.endpoint="${AFFINE_INDEXER_SEARCH_ENDPOINT}"`,
|
|
`--set-string global.indexer.username="${AFFINE_INDEXER_SEARCH_USERNAME}"`,
|
|
`--set-string global.indexer.password="${AFFINE_INDEXER_SEARCH_PASSWORD}"`,
|
|
];
|
|
const serviceAnnotations = [
|
|
`--set-json web.serviceAccount.annotations="{ \\"iam.gke.io/gcp-service-account\\": \\"${APP_IAM_ACCOUNT}\\" }"`,
|
|
`--set-json graphql.serviceAccount.annotations="{ \\"iam.gke.io/gcp-service-account\\": \\"${APP_IAM_ACCOUNT}\\" }"`,
|
|
`--set-json sync.serviceAccount.annotations="{ \\"iam.gke.io/gcp-service-account\\": \\"${APP_IAM_ACCOUNT}\\" }"`,
|
|
`--set-json doc.serviceAccount.annotations="{ \\"iam.gke.io/gcp-service-account\\": \\"${APP_IAM_ACCOUNT}\\" }"`,
|
|
].concat(
|
|
isProduction || isBeta || isInternal
|
|
? [
|
|
`--set-json web.service.annotations="{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }"`,
|
|
`--set-json graphql.service.annotations="{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }"`,
|
|
`--set-json sync.service.annotations="{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }"`,
|
|
`--set-json cloud-sql-proxy.serviceAccount.annotations="{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_SQL_IAM_ACCOUNT}\\" }"`,
|
|
`--set-json cloud-sql-proxy.nodeSelector="{ \\"iam.gke.io/gke-metadata-server-enabled\\": \\"true\\" }"`,
|
|
]
|
|
: []
|
|
);
|
|
|
|
const cpu = cpuConfig[buildType];
|
|
const resources = cpu
|
|
? [
|
|
`--set web.resources.requests.cpu="${cpu.web}"`,
|
|
`--set graphql.resources.requests.cpu="${cpu.graphql}"`,
|
|
`--set sync.resources.requests.cpu="${cpu.sync}"`,
|
|
`--set doc.resources.requests.cpu="${cpu.doc}"`,
|
|
]
|
|
: [];
|
|
|
|
const replica = replicaConfig[buildType] || replicaConfig.canary;
|
|
|
|
const namespace = isProduction
|
|
? 'production'
|
|
: isBeta
|
|
? 'beta'
|
|
: isInternal
|
|
? 'internal'
|
|
: 'dev';
|
|
|
|
const host = DEPLOY_HOST || CANARY_DEPLOY_HOST;
|
|
const deployCommand = [
|
|
`helm upgrade --install affine .github/helm/affine`,
|
|
`--namespace ${namespace}`,
|
|
`--set-string global.deployment.type="affine"`,
|
|
`--set-string global.deployment.platform="gcp"`,
|
|
`--set-string global.app.buildType="${buildType}"`,
|
|
`--set global.ingress.enabled=true`,
|
|
`--set-json global.ingress.annotations="{ \\"kubernetes.io/ingress.class\\": \\"gce\\", \\"kubernetes.io/ingress.allow-http\\": \\"true\\", \\"kubernetes.io/ingress.global-static-ip-name\\": \\"${STATIC_IP_NAME}\\" }"`,
|
|
`--set-string global.ingress.host="${host}"`,
|
|
`--set-string global.version="${APP_VERSION}"`,
|
|
...redisAndPostgres,
|
|
...indexerOptions,
|
|
`--set web.replicaCount=${replica.web}`,
|
|
`--set-string web.image.tag="${imageTag}"`,
|
|
`--set graphql.replicaCount=${replica.graphql}`,
|
|
`--set-string graphql.image.tag="${imageTag}"`,
|
|
`--set graphql.app.host=${host}`,
|
|
`--set sync.replicaCount=${replica.sync}`,
|
|
`--set-string sync.image.tag="${imageTag}"`,
|
|
`--set-string renderer.image.tag="${imageTag}"`,
|
|
`--set renderer.app.host=${host}`,
|
|
`--set renderer.replicaCount=${replica.renderer}`,
|
|
`--set-string doc.image.tag="${imageTag}"`,
|
|
`--set doc.app.host=${host}`,
|
|
`--set doc.replicaCount=${replica.doc}`,
|
|
...serviceAnnotations,
|
|
...resources,
|
|
`--timeout 10m`,
|
|
flag,
|
|
].join(' ');
|
|
return deployCommand;
|
|
};
|
|
|
|
const output = execSync(createHelmCommand({ isDryRun: true }), {
|
|
encoding: 'utf-8',
|
|
stdio: ['inherit', 'pipe', 'inherit'],
|
|
});
|
|
const templates = output
|
|
.split('---')
|
|
.filter(yml => !yml.split('\n').some(line => line.trim() === 'kind: Secret'))
|
|
.join('---');
|
|
console.log(templates);
|
|
|
|
execSync(createHelmCommand({ isDryRun: false }), {
|
|
encoding: 'utf-8',
|
|
stdio: 'inherit',
|
|
});
|