Skip to content

Commit 0e96b42

Browse files
Esievesunlei
Esieve
authored and
sunlei
committed
feat: BigQuery
1 parent c0b2c9f commit 0e96b42

8 files changed

Lines changed: 217 additions & 2 deletions

File tree

‎server/node-service/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
"@aws-sdk/client-lambda": "^3.272.0",
3535
"@aws-sdk/client-s3": "^3.238.0",
3636
"@aws-sdk/s3-request-presigner": "^3.241.0",
37+
"@google-cloud/bigquery": "^6.1.0",
3738
"@google-cloud/storage": "^6.9.3",
3839
"@supabase/supabase-js": "^2.10.0",
3940
"@types/axios": "^0.14.0",

‎server/node-service/src/plugins/bigQuery/dataSourceConfig.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import { ConfigToType } from "openblocks-sdk/dataSource";
2+
3+
export const dataSourceConfig = {
4+
type: "dataSource",
5+
params: [
6+
{
7+
key: "privateKey",
8+
label: "Private Key",
9+
type: "password",
10+
tooltip:
11+
"The private key associated with a Service Account with Big Query privileges, [Documentation](https://cloud.google.com/iam/docs/service-accounts) for service accounts.",
12+
rules: [
13+
{ required: true, message: "Please input your private key of google Service Account" },
14+
],
15+
},
16+
{
17+
key: "region",
18+
type: "textInput",
19+
label: "Region",
20+
tooltip: '[Documentation](https://cloud.google.com/bigquery/docs/locations)'
21+
}
22+
],
23+
} as const;
24+
25+
export type DataSourceDataType = ConfigToType<typeof dataSourceConfig>;

‎server/node-service/src/plugins/bigQuery/index.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import { PluginContext } from "openblocks-sdk/dataSource";
2+
import queryConfig, { ActionDataType } from "./queryConfig";
3+
import { dataSourceConfig, DataSourceDataType } from "./dataSourceConfig";
4+
import run, { validateDataSourceConfig } from "./run";
5+
6+
const bigQueryPlugin = {
7+
id: "bigQuery",
8+
name: "Big Query",
9+
icon: "bigQuery.svg",
10+
category: "database",
11+
dataSourceConfig,
12+
queryConfig: queryConfig,
13+
14+
validateDataSourceConfig: async (dataSourceConfig: DataSourceDataType) => {
15+
return validateDataSourceConfig(dataSourceConfig);
16+
},
17+
18+
run: async (action: ActionDataType, dataSourceConfig: DataSourceDataType, ctx: PluginContext) => {
19+
try {
20+
return await run(action, dataSourceConfig);
21+
} catch (e) {
22+
throw e;
23+
}
24+
},
25+
};
26+
27+
export default bigQueryPlugin;

‎server/node-service/src/plugins/bigQuery/queryConfig.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import { ConfigToType } from "openblocks-sdk/dataSource";
2+
3+
const queryConfig = {
4+
type: "query",
5+
label: "Action",
6+
actions: [
7+
{
8+
actionName: "Query",
9+
label: "Query",
10+
params: [
11+
{
12+
label: "Google SQL",
13+
key: "googleSQL",
14+
type: "sqlInput",
15+
},
16+
],
17+
},
18+
],
19+
} as const;
20+
21+
export type ActionDataType = ConfigToType<typeof queryConfig>;
22+
23+
export default queryConfig;

‎server/node-service/src/plugins/bigQuery/run.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import { DataSourceDataType } from "./dataSourceConfig";
2+
import { ActionDataType } from "./queryConfig";
3+
import { BigQuery } from "@google-cloud/bigquery";
4+
import log from "loglevel";
5+
6+
function getClient(params: DataSourceDataType) {
7+
return new BigQuery({
8+
credentials: JSON.parse(params.privateKey),
9+
location: params.region
10+
});
11+
}
12+
13+
export async function validateDataSourceConfig(dataSourceConfig: DataSourceDataType) {
14+
try {
15+
const client = getClient(dataSourceConfig);
16+
await client.getDatasets();
17+
return {
18+
success: true,
19+
};
20+
} catch (e) {
21+
throw e;
22+
}
23+
}
24+
25+
export default async function run(action: ActionDataType, dataSourceConfig: DataSourceDataType) {
26+
const client = getClient(dataSourceConfig);
27+
if (action.actionName === 'Query') {
28+
const [results] = await client.query(action.googleSQL)
29+
return results;
30+
}
31+
}

‎server/node-service/src/plugins/googleCloudStorage/dataSourceConfig.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ export const dataSourceConfig = {
88
label: "Private Key",
99
type: "password",
1010
tooltip:
11-
"The private key associated with a Service Account with GCS privileges, [Documentation](https://cloud.google.com/iam/docs/service-accounts?hl=zh-cn) for service accounts.",
11+
"The private key associated with a Service Account with GCS privileges, [Documentation](https://cloud.google.com/iam/docs/service-accounts) for service accounts.",
1212
rules: [
1313
{ required: true, message: "Please input your private key of google Service Account" },
1414
],
@@ -22,3 +22,4 @@ export const dataSourceConfig = {
2222
} as const;
2323

2424
export type DataSourceDataType = ConfigToType<typeof dataSourceConfig>;
25+

‎server/node-service/src/plugins/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import gitlabPlugin from "./gitlab";
3030
import faunaPlugin from "./fauna";
3131
import huggingFaceInferencePlugin from "./huggingFaceInference";
3232
import didPlugin from "./did";
33+
import bigQueryPlugin from "./bigQuery";
3334

3435
let plugins: (DataSourcePlugin | DataSourcePluginFactory)[] = [
3536
s3Plugin,
@@ -55,7 +56,6 @@ let plugins: (DataSourcePlugin | DataSourcePluginFactory)[] = [
5556
sendGridPlugin,
5657
shopifyPlugin,
5758
slackPlugin,
58-
stripePlugin,
5959
supabasePlugin,
6060
cloudinaryPlugin,
6161
notionPlugin,
@@ -64,6 +64,7 @@ let plugins: (DataSourcePlugin | DataSourcePluginFactory)[] = [
6464
gitlabPlugin,
6565
faunaPlugin,
6666
didPlugin,
67+
bigQueryPlugin
6768
];
6869

6970
try {

‎server/node-service/yarn.lock‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3965,6 +3965,43 @@ __metadata:
39653965
languageName: node
39663966
linkType: hard
39673967

3968+
"@google-cloud/bigquery@npm:^6.1.0":
3969+
version: 6.1.0
3970+
resolution: "@google-cloud/bigquery@npm:6.1.0"
3971+
dependencies:
3972+
"@google-cloud/common": ^4.0.0
3973+
"@google-cloud/paginator": ^4.0.0
3974+
"@google-cloud/promisify": ^3.0.0
3975+
arrify: ^2.0.1
3976+
big.js: ^6.0.0
3977+
duplexify: ^4.0.0
3978+
extend: ^3.0.2
3979+
is: ^3.3.0
3980+
p-event: ^4.1.0
3981+
readable-stream: ^4.0.0
3982+
stream-events: ^1.0.5
3983+
uuid: ^9.0.0
3984+
checksum: a6b9aa30fe5e6ab1f3b3523aa46392f2c7606c4892f5537b0a2dd21b96cf529586841078c868c1a5a4d48ba4105451ca611d7a2f429f1f31dcc588513aadf3ff
3985+
languageName: node
3986+
linkType: hard
3987+
3988+
"@google-cloud/common@npm:^4.0.0":
3989+
version: 4.0.2
3990+
resolution: "@google-cloud/common@npm:4.0.2"
3991+
dependencies:
3992+
"@google-cloud/projectify": ^3.0.0
3993+
"@google-cloud/promisify": ^3.0.0
3994+
arrify: ^2.0.1
3995+
duplexify: ^4.1.1
3996+
ent: ^2.2.0
3997+
extend: ^3.0.2
3998+
google-auth-library: ^8.0.2
3999+
retry-request: ^5.0.0
4000+
teeny-request: ^8.0.0
4001+
checksum: 0bc81478069ae61db9558c8582e0553b981a9fdefce0f3e71e8ff3913c9666892b38eaaf1dfcf9ad695270558e5a8110267e7ab4d7645033e96c97b948422cda
4002+
languageName: node
4003+
linkType: hard
4004+
39684005
"@google-cloud/firestore@npm:^6.4.0":
39694006
version: 6.4.2
39704007
resolution: "@google-cloud/firestore@npm:6.4.2::__archiveUrl=https%3A%2F%2Fregistry.npmjs.org%2F%40google-cloud%2Ffirestore%2F-%2Ffirestore-6.4.2.tgz"
@@ -3987,6 +4024,16 @@ __metadata:
39874024
languageName: node
39884025
linkType: hard
39894026

4027+
"@google-cloud/paginator@npm:^4.0.0":
4028+
version: 4.0.1
4029+
resolution: "@google-cloud/paginator@npm:4.0.1"
4030+
dependencies:
4031+
arrify: ^2.0.0
4032+
extend: ^3.0.2
4033+
checksum: 40ecfb59512ddbb76ca377cb96b61673d8d210397723dcaac41d8a553264bf0c09d3754db25dd3c476f8d85941b5017cc158b4e81c8c6a054aea020c32a1e4ba
4034+
languageName: node
4035+
linkType: hard
4036+
39904037
"@google-cloud/projectify@npm:^3.0.0":
39914038
version: 3.0.0
39924039
resolution: "@google-cloud/projectify@npm:3.0.0::__archiveUrl=https%3A%2F%2Fregistry.npmjs.org%2F%40google-cloud%2Fprojectify%2F-%2Fprojectify-3.0.0.tgz"
@@ -5277,6 +5324,13 @@ __metadata:
52775324
languageName: node
52785325
linkType: hard
52795326

5327+
"arrify@npm:^2.0.1":
5328+
version: 2.0.1
5329+
resolution: "arrify@npm:2.0.1"
5330+
checksum: 067c4c1afd182806a82e4c1cb8acee16ab8b5284fbca1ce29408e6e91281c36bb5b612f6ddfbd40a0f7a7e0c75bf2696eb94c027f6e328d6e9c52465c98e4209
5331+
languageName: node
5332+
linkType: hard
5333+
52805334
"ast-types@npm:^0.13.2":
52815335
version: 0.13.4
52825336
resolution: "ast-types@npm:0.13.4"
@@ -5433,6 +5487,13 @@ __metadata:
54335487
languageName: node
54345488
linkType: hard
54355489

5490+
"big.js@npm:^6.0.0":
5491+
version: 6.2.1
5492+
resolution: "big.js@npm:6.2.1"
5493+
checksum: 0b234a2fd56c52bed2798ed2020bcab6fef5e9523b99a05406ad071d1aed6ee97ada9fb8de9576092da74c68825c276e19015743b8d1baea269b60a5c666b0cd
5494+
languageName: node
5495+
linkType: hard
5496+
54365497
"bignumber.js@npm:^9.0.0":
54375498
version: 9.1.1
54385499
resolution: "bignumber.js@npm:9.1.1::__archiveUrl=https%3A%2F%2Fregistry.npmjs.org%2Fbignumber.js%2F-%2Fbignumber.js-9.1.1.tgz"
@@ -6226,6 +6287,18 @@ __metadata:
62266287
languageName: node
62276288
linkType: hard
62286289

6290+
"duplexify@npm:^4.1.1":
6291+
version: 4.1.2
6292+
resolution: "duplexify@npm:4.1.2"
6293+
dependencies:
6294+
end-of-stream: ^1.4.1
6295+
inherits: ^2.0.3
6296+
readable-stream: ^3.1.1
6297+
stream-shift: ^1.0.0
6298+
checksum: 964376c61c0e92f6ed0694b3ba97c84f199413dc40ab8dfdaef80b7a7f4982fcabf796214e28ed614a5bc1ec45488a29b81e7d46fa3f5ddf65bcb118c20145ad
6299+
languageName: node
6300+
linkType: hard
6301+
62296302
"dynamodb-data-types@npm:^4.0.1":
62306303
version: 4.0.1
62316304
resolution: "dynamodb-data-types@npm:4.0.1::__archiveUrl=https%3A%2F%2Fregistry.npmjs.org%2Fdynamodb-data-types%2F-%2Fdynamodb-data-types-4.0.1.tgz"
@@ -7520,6 +7593,13 @@ __metadata:
75207593
languageName: node
75217594
linkType: hard
75227595

7596+
"is@npm:^3.3.0":
7597+
version: 3.3.0
7598+
resolution: "is@npm:3.3.0"
7599+
checksum: 81fad3b40c606984c2d0699207c4c48d2a0d29cc834b274d0b74c172f3eeebdb981301fe0d690ce090a96bf021a8a1f8b1325262ad9870c525e557ac4a559c56
7600+
languageName: node
7601+
linkType: hard
7602+
75237603
"isarray@npm:0.0.1":
75247604
version: 0.0.1
75257605
resolution: "isarray@npm:0.0.1"
@@ -9131,6 +9211,22 @@ __metadata:
91319211
languageName: node
91329212
linkType: hard
91339213

9214+
"p-event@npm:^4.1.0":
9215+
version: 4.2.0
9216+
resolution: "p-event@npm:4.2.0"
9217+
dependencies:
9218+
p-timeout: ^3.1.0
9219+
checksum: 8a3588f7a816a20726a3262dfeee70a631e3997e4773d23219176333eda55cce9a76219e3d2b441b331eb746e14fdb381eb2694ab9ff2fcf87c846462696fe89
9220+
languageName: node
9221+
linkType: hard
9222+
9223+
"p-finally@npm:^1.0.0":
9224+
version: 1.0.0
9225+
resolution: "p-finally@npm:1.0.0"
9226+
checksum: 93a654c53dc805dd5b5891bab16eb0ea46db8f66c4bfd99336ae929323b1af2b70a8b0654f8f1eae924b2b73d037031366d645f1fd18b3d30cbd15950cc4b1d4
9227+
languageName: node
9228+
linkType: hard
9229+
91349230
"p-limit@npm:^2.2.0":
91359231
version: 2.3.0
91369232
resolution: "p-limit@npm:2.3.0::__archiveUrl=https%3A%2F%2Fregistry.npmjs.org%2Fp-limit%2F-%2Fp-limit-2.3.0.tgz"
@@ -9167,6 +9263,15 @@ __metadata:
91679263
languageName: node
91689264
linkType: hard
91699265

9266+
"p-timeout@npm:^3.1.0":
9267+
version: 3.2.0
9268+
resolution: "p-timeout@npm:3.2.0"
9269+
dependencies:
9270+
p-finally: ^1.0.0
9271+
checksum: 3dd0eaa048780a6f23e5855df3dd45c7beacff1f820476c1d0d1bcd6648e3298752ba2c877aa1c92f6453c7dd23faaf13d9f5149fc14c0598a142e2c5e8d649c
9272+
languageName: node
9273+
linkType: hard
9274+
91709275
"p-try@npm:^2.0.0":
91719276
version: 2.2.0
91729277
resolution: "p-try@npm:2.2.0::__archiveUrl=https%3A%2F%2Fregistry.npmjs.org%2Fp-try%2F-%2Fp-try-2.2.0.tgz"
@@ -10280,6 +10385,7 @@ __metadata:
1028010385
"@aws-sdk/client-lambda": ^3.272.0
1028110386
"@aws-sdk/client-s3": ^3.238.0
1028210387
"@aws-sdk/s3-request-presigner": ^3.241.0
10388+
"@google-cloud/bigquery": ^6.1.0
1028310389
"@google-cloud/storage": ^6.9.3
1028410390
"@supabase/supabase-js": ^2.10.0
1028510391
"@types/axios": ^0.14.0

0 commit comments

Comments
 (0)