blob: c48d9ba4101afcb87d8082bdc2e6981ebabd0bb5 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/* Fluent Bit
* ==========
* Copyright (C) 2015-2022 The Fluent Bit Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FLB_OUT_BIGQUERY
#define FLB_OUT_BIGQUERY
#include <fluent-bit/flb_info.h>
#include <fluent-bit/flb_output.h>
#include <fluent-bit/flb_oauth2.h>
#include <fluent-bit/flb_sds.h>
/* refresh token every 50 minutes */
#define FLB_BIGQUERY_TOKEN_REFRESH 3000
/* BigQuery streaming inserts oauth scope */
#define FLB_BIGQUERY_SCOPE "https://www.googleapis.com/auth/bigquery.insertdata"
/* BigQuery authorization URL */
#define FLB_BIGQUERY_AUTH_URL "https://oauth2.googleapis.com/token"
#define FLB_BIGQUERY_RESOURCE_TEMPLATE "/bigquery/v2/projects/%s/datasets/%s/tables/%s/insertAll"
#define FLB_BIGQUERY_URL_BASE "https://www.googleapis.com"
#define FLB_BIGQUERY_GOOGLE_STS_URL "https://sts.googleapis.com"
#define FLB_BIGQUERY_GOOGLE_IAM_URL "https://iamcredentials.googleapis.com"
#define FLB_BIGQUERY_AWS_STS_ENDPOINT "/?Action=GetCallerIdentity&Version=2011-06-15"
#define FLB_BIGQUERY_GOOGLE_CLOUD_TARGET_RESOURCE \
"//iam.googleapis.com/projects/%s/locations/global/workloadIdentityPools/%s/providers/%s"
#define FLB_BIGQUERY_GOOGLE_STS_TOKEN_GRANT_TYPE "urn:ietf:params:oauth:grant-type:token-exchange"
#define FLB_BIGQUERY_GOOGLE_STS_TOKEN_REQUESTED_TOKEN_TYPE "urn:ietf:params:oauth:token-type:access_token"
#define FLB_BIGQUERY_GOOGLE_STS_TOKEN_SCOPE "https://www.googleapis.com/auth/cloud-platform"
#define FLB_BIGQUERY_GOOGLE_STS_TOKEN_SUBJECT_TOKEN_TYPE "urn:ietf:params:aws:token-type:aws4_request"
#define FLB_BIGQUERY_GOOGLE_CLOUD_TOKEN_ENDPOINT "/v1/token"
#define FLB_BIGQUERY_GOOGLE_GEN_ACCESS_TOKEN_REQUEST_BODY \
"{\"scope\": [\"https://www.googleapis.com/auth/cloud-platform\"]}"
#define FLB_BIGQUERY_GOOGLE_GEN_ACCESS_TOKEN_URL \
"https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/%s:generateAccessToken"
struct flb_bigquery_oauth_credentials {
/* parsed credentials file */
flb_sds_t type;
flb_sds_t project_id;
flb_sds_t private_key_id;
flb_sds_t private_key;
flb_sds_t client_email;
flb_sds_t client_id;
flb_sds_t auth_uri;
flb_sds_t token_uri;
};
struct flb_bigquery {
/* credentials */
flb_sds_t credentials_file;
struct flb_bigquery_oauth_credentials *oauth_credentials;
/* Workload Identity Federation */
int has_identity_federation;
flb_sds_t project_number;
flb_sds_t pool_id;
flb_sds_t provider_id;
flb_sds_t aws_region;
flb_sds_t google_service_account;
/* AWS IMDS */
struct flb_tls *aws_tls;
struct flb_aws_provider *aws_provider;
/* AWS STS */
flb_sds_t aws_sts_endpoint;
struct flb_tls *aws_sts_tls;
struct flb_upstream *aws_sts_upstream;
/* Google STS API */
struct flb_tls *google_sts_tls;
struct flb_upstream *google_sts_upstream;
/* Google Service Account Credentials API */
struct flb_tls *google_iam_tls;
struct flb_upstream *google_iam_upstream;
/* Google OAuth access token for service account, that was exchanged for AWS credentials */
flb_sds_t sa_token;
time_t sa_token_expiry;
/* bigquery configuration */
flb_sds_t project_id;
flb_sds_t dataset_id;
flb_sds_t table_id;
int skip_invalid_rows;
int ignore_unknown_values;
flb_sds_t uri;
/* oauth2 context */
struct flb_oauth2 *o;
/* mutex for acquiring oauth tokens */
pthread_mutex_t token_mutex;
/* Upstream connection to the backend server */
struct flb_upstream *u;
/* Fluent Bit context */
struct flb_config *config;
/* Plugin output instance reference */
struct flb_output_instance *ins;
};
#endif
|
