Skip to content

Commit 9667ef1

Browse files
arithmetic1728arithmetic1728
authored
feat: regenerate the client lib to pick new mtls env (googleapis#197)
* feat: regenerate the client lib to pick new mtls env * update google-api-core version * revert some files in samples/ and scripts/
1 parent 89c671a commit 9667ef1

20 files changed

Lines changed: 881 additions & 678 deletions

File tree

‎.kokoro/populate-secrets.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
# Copyright 2020 Google LLC.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -eo pipefail
17+
18+
function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;}
19+
function msg { println "$*" >&2 ;}
20+
function println { printf '%s\n' "$(now) $*" ;}
21+
22+
23+
# Populates requested secrets set in SECRET_MANAGER_KEYS from service account:
24+
# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com
25+
SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
26+
msg "Creating folder on disk for secrets: ${SECRET_LOCATION}"
27+
mkdir -p ${SECRET_LOCATION}
28+
for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g")
29+
do
30+
msg "Retrieving secret ${key}"
31+
docker run --entrypoint=gcloud \
32+
--volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \
33+
gcr.io/google.com/cloudsdktool/cloud-sdk \
34+
secrets versions access latest \
35+
--project cloud-devrel-kokoro-resources \
36+
--secret ${key} > \
37+
"${SECRET_LOCATION}/${key}"
38+
if [[ $? == 0 ]]; then
39+
msg "Secret written to ${SECRET_LOCATION}/${key}"
40+
else
41+
msg "Error retrieving secret ${key}"
42+
fi
43+
done

‎.kokoro/release/common.cfg‎

Lines changed: 13 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -23,42 +23,18 @@ env_vars: {
2323
value: "github/python-pubsub/.kokoro/release.sh"
2424
}
2525

26-
# Fetch the token needed for reporting release status to GitHub
27-
before_action {
28-
fetch_keystore {
29-
keystore_resource {
30-
keystore_config_id: 73713
31-
keyname: "yoshi-automation-github-key"
32-
}
33-
}
34-
}
35-
36-
# Fetch PyPI password
37-
before_action {
38-
fetch_keystore {
39-
keystore_resource {
40-
keystore_config_id: 73713
41-
keyname: "google_cloud_pypi_password"
42-
}
43-
}
44-
}
45-
46-
# Fetch magictoken to use with Magic Github Proxy
47-
before_action {
48-
fetch_keystore {
49-
keystore_resource {
50-
keystore_config_id: 73713
51-
keyname: "releasetool-magictoken"
52-
}
53-
}
26+
# Fetch PyPI password
27+
before_action {
28+
fetch_keystore {
29+
keystore_resource {
30+
keystore_config_id: 73713
31+
keyname: "google_cloud_pypi_password"
32+
}
33+
}
5434
}
5535

56-
# Fetch api key to use with Magic Github Proxy
57-
before_action {
58-
fetch_keystore {
59-
keystore_resource {
60-
keystore_config_id: 73713
61-
keyname: "magic-github-proxy-api-key"
62-
}
63-
}
64-
}
36+
# Tokens needed to report release status back to GitHub
37+
env_vars: {
38+
key: "SECRET_MANAGER_KEYS"
39+
value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem"
40+
}

‎.kokoro/trampoline.sh‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,14 @@
1515

1616
set -eo pipefail
1717

18-
python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" || ret_code=$?
18+
# Always run the cleanup script, regardless of the success of bouncing into
19+
# the container.
20+
function cleanup() {
21+
chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
22+
${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
23+
echo "cleanup";
24+
}
25+
trap cleanup EXIT
1926

20-
chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
21-
${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true
22-
23-
exit ${ret_code}
27+
$(dirname $0)/populate-secrets.sh # Secret Manager secrets.
28+
python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py"

‎docs/conf.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
# -- General configuration ------------------------------------------------
3030

3131
# If your documentation needs a minimal Sphinx version, state it here.
32-
needs_sphinx = "1.6.3"
32+
needs_sphinx = "1.5.5"
3333

3434
# Add any Sphinx extension module names here, as strings. They can be
3535
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom

‎google/pubsub_v1/services/publisher/async_client.py‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ class PublisherAsyncClient:
4949
DEFAULT_MTLS_ENDPOINT = PublisherClient.DEFAULT_MTLS_ENDPOINT
5050

5151
topic_path = staticmethod(PublisherClient.topic_path)
52+
parse_topic_path = staticmethod(PublisherClient.parse_topic_path)
5253

5354
from_service_account_file = PublisherClient.from_service_account_file
5455
from_service_account_json = from_service_account_file
@@ -79,16 +80,19 @@ def __init__(
7980
client_options (ClientOptions): Custom options for the client. It
8081
won't take effect if a ``transport`` instance is provided.
8182
(1) The ``api_endpoint`` property can be used to override the
82-
default endpoint provided by the client. GOOGLE_API_USE_MTLS
83+
default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT
8384
environment variable can also be used to override the endpoint:
8485
"always" (always use the default mTLS endpoint), "never" (always
85-
use the default regular endpoint, this is the default value for
86-
the environment variable) and "auto" (auto switch to the default
87-
mTLS endpoint if client SSL credentials is present). However,
88-
the ``api_endpoint`` property takes precedence if provided.
89-
(2) The ``client_cert_source`` property is used to provide client
90-
SSL credentials for mutual TLS transport. If not provided, the
91-
default SSL credentials will be used if present.
86+
use the default regular endpoint) and "auto" (auto switch to the
87+
default mTLS endpoint if client certificate is present, this is
88+
the default value). However, the ``api_endpoint`` property takes
89+
precedence if provided.
90+
(2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable
91+
is "true", then the ``client_cert_source`` property can be used
92+
to provide client certificate for mutual TLS transport. If
93+
not provided, the default SSL client certificate will be used if
94+
present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not
95+
set, no client certificate will be used.
9296
9397
Raises:
9498
google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport
@@ -308,13 +312,13 @@ async def publish(
308312
maximum=60.0,
309313
multiplier=1.3,
310314
predicate=retries.if_exception_type(
311-
exceptions.Aborted,
312315
exceptions.Cancelled,
313316
exceptions.DeadlineExceeded,
317+
exceptions.InternalServerError,
314318
exceptions.ResourceExhausted,
315319
exceptions.ServiceUnavailable,
316320
exceptions.Unknown,
317-
exceptions.InternalServerError,
321+
exceptions.Aborted,
318322
),
319323
),
320324
default_timeout=60.0,
@@ -391,8 +395,8 @@ async def get_topic(
391395
multiplier=1.3,
392396
predicate=retries.if_exception_type(
393397
exceptions.Aborted,
394-
exceptions.ServiceUnavailable,
395398
exceptions.Unknown,
399+
exceptions.ServiceUnavailable,
396400
),
397401
),
398402
default_timeout=60.0,
@@ -473,8 +477,8 @@ async def list_topics(
473477
multiplier=1.3,
474478
predicate=retries.if_exception_type(
475479
exceptions.Aborted,
476-
exceptions.ServiceUnavailable,
477480
exceptions.Unknown,
481+
exceptions.ServiceUnavailable,
478482
),
479483
),
480484
default_timeout=60.0,
@@ -564,8 +568,8 @@ async def list_topic_subscriptions(
564568
multiplier=1.3,
565569
predicate=retries.if_exception_type(
566570
exceptions.Aborted,
567-
exceptions.ServiceUnavailable,
568571
exceptions.Unknown,
572+
exceptions.ServiceUnavailable,
569573
),
570574
),
571575
default_timeout=60.0,
@@ -659,8 +663,8 @@ async def list_topic_snapshots(
659663
multiplier=1.3,
660664
predicate=retries.if_exception_type(
661665
exceptions.Aborted,
662-
exceptions.ServiceUnavailable,
663666
exceptions.Unknown,
667+
exceptions.ServiceUnavailable,
664668
),
665669
),
666670
default_timeout=60.0,

‎google/pubsub_v1/services/publisher/client.py‎

Lines changed: 46 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#
1717

1818
from collections import OrderedDict
19+
from distutils import util
1920
import os
2021
import re
2122
from typing import Callable, Dict, Sequence, Tuple, Type, Union
@@ -27,6 +28,7 @@
2728
from google.api_core import retry as retries # type: ignore
2829
from google.auth import credentials # type: ignore
2930
from google.auth.transport import mtls # type: ignore
31+
from google.auth.transport.grpc import SslCredentials # type: ignore
3032
from google.auth.exceptions import MutualTLSChannelError # type: ignore
3133
from google.oauth2 import service_account # type: ignore
3234

@@ -177,16 +179,19 @@ def __init__(
177179
client_options (ClientOptions): Custom options for the client. It
178180
won't take effect if a ``transport`` instance is provided.
179181
(1) The ``api_endpoint`` property can be used to override the
180-
default endpoint provided by the client. GOOGLE_API_USE_MTLS
182+
default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT
181183
environment variable can also be used to override the endpoint:
182184
"always" (always use the default mTLS endpoint), "never" (always
183-
use the default regular endpoint, this is the default value for
184-
the environment variable) and "auto" (auto switch to the default
185-
mTLS endpoint if client SSL credentials is present). However,
186-
the ``api_endpoint`` property takes precedence if provided.
187-
(2) The ``client_cert_source`` property is used to provide client
188-
SSL credentials for mutual TLS transport. If not provided, the
189-
default SSL credentials will be used if present.
185+
use the default regular endpoint) and "auto" (auto switch to the
186+
default mTLS endpoint if client certificate is present, this is
187+
the default value). However, the ``api_endpoint`` property takes
188+
precedence if provided.
189+
(2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable
190+
is "true", then the ``client_cert_source`` property can be used
191+
to provide client certificate for mutual TLS transport. If
192+
not provided, the default SSL client certificate will be used if
193+
present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not
194+
set, no client certificate will be used.
190195
client_info (google.api_core.gapic_v1.client_info.ClientInfo):
191196
The client info used to send a user-agent string along with
192197
API requests. If ``None``, then default info will be used.
@@ -202,25 +207,43 @@ def __init__(
202207
if client_options is None:
203208
client_options = ClientOptions.ClientOptions()
204209

205-
if client_options.api_endpoint is None:
206-
use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS", "never")
210+
# Create SSL credentials for mutual TLS if needed.
211+
use_client_cert = bool(
212+
util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"))
213+
)
214+
215+
ssl_credentials = None
216+
is_mtls = False
217+
if use_client_cert:
218+
if client_options.client_cert_source:
219+
import grpc # type: ignore
220+
221+
cert, key = client_options.client_cert_source()
222+
ssl_credentials = grpc.ssl_channel_credentials(
223+
certificate_chain=cert, private_key=key
224+
)
225+
is_mtls = True
226+
else:
227+
creds = SslCredentials()
228+
is_mtls = creds.is_mtls
229+
ssl_credentials = creds.ssl_credentials if is_mtls else None
230+
231+
# Figure out which api endpoint to use.
232+
if client_options.api_endpoint is not None:
233+
api_endpoint = client_options.api_endpoint
234+
else:
235+
use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto")
207236
if use_mtls_env == "never":
208-
client_options.api_endpoint = self.DEFAULT_ENDPOINT
237+
api_endpoint = self.DEFAULT_ENDPOINT
209238
elif use_mtls_env == "always":
210-
client_options.api_endpoint = self.DEFAULT_MTLS_ENDPOINT
239+
api_endpoint = self.DEFAULT_MTLS_ENDPOINT
211240
elif use_mtls_env == "auto":
212-
has_client_cert_source = (
213-
client_options.client_cert_source is not None
214-
or mtls.has_default_client_cert_source()
215-
)
216-
client_options.api_endpoint = (
217-
self.DEFAULT_MTLS_ENDPOINT
218-
if has_client_cert_source
219-
else self.DEFAULT_ENDPOINT
241+
api_endpoint = (
242+
self.DEFAULT_MTLS_ENDPOINT if is_mtls else self.DEFAULT_ENDPOINT
220243
)
221244
else:
222245
raise MutualTLSChannelError(
223-
"Unsupported GOOGLE_API_USE_MTLS value. Accepted values: never, auto, always"
246+
"Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted values: never, auto, always"
224247
)
225248

226249
# Save or instantiate the transport.
@@ -244,10 +267,9 @@ def __init__(
244267
self._transport = Transport(
245268
credentials=credentials,
246269
credentials_file=client_options.credentials_file,
247-
host=client_options.api_endpoint,
270+
host=api_endpoint,
248271
scopes=client_options.scopes,
249-
api_mtls_endpoint=client_options.api_endpoint,
250-
client_cert_source=client_options.client_cert_source,
272+
ssl_channel_credentials=ssl_credentials,
251273
quota_project_id=client_options.quota_project_id,
252274
client_info=client_info,
253275
)

‎google/pubsub_v1/services/publisher/transports/base.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,13 @@ def _prep_wrapped_messages(self, client_info):
139139
maximum=60.0,
140140
multiplier=1.3,
141141
predicate=retries.if_exception_type(
142-
exceptions.Aborted,
143142
exceptions.Cancelled,
144143
exceptions.DeadlineExceeded,
144+
exceptions.InternalServerError,
145145
exceptions.ResourceExhausted,
146146
exceptions.ServiceUnavailable,
147147
exceptions.Unknown,
148-
exceptions.InternalServerError,
148+
exceptions.Aborted,
149149
),
150150
),
151151
default_timeout=60.0,
@@ -159,8 +159,8 @@ def _prep_wrapped_messages(self, client_info):
159159
multiplier=1.3,
160160
predicate=retries.if_exception_type(
161161
exceptions.Aborted,
162-
exceptions.ServiceUnavailable,
163162
exceptions.Unknown,
163+
exceptions.ServiceUnavailable,
164164
),
165165
),
166166
default_timeout=60.0,
@@ -174,8 +174,8 @@ def _prep_wrapped_messages(self, client_info):
174174
multiplier=1.3,
175175
predicate=retries.if_exception_type(
176176
exceptions.Aborted,
177-
exceptions.ServiceUnavailable,
178177
exceptions.Unknown,
178+
exceptions.ServiceUnavailable,
179179
),
180180
),
181181
default_timeout=60.0,
@@ -189,8 +189,8 @@ def _prep_wrapped_messages(self, client_info):
189189
multiplier=1.3,
190190
predicate=retries.if_exception_type(
191191
exceptions.Aborted,
192-
exceptions.ServiceUnavailable,
193192
exceptions.Unknown,
193+
exceptions.ServiceUnavailable,
194194
),
195195
),
196196
default_timeout=60.0,
@@ -204,8 +204,8 @@ def _prep_wrapped_messages(self, client_info):
204204
multiplier=1.3,
205205
predicate=retries.if_exception_type(
206206
exceptions.Aborted,
207-
exceptions.ServiceUnavailable,
208207
exceptions.Unknown,
208+
exceptions.ServiceUnavailable,
209209
),
210210
),
211211
default_timeout=60.0,

0 commit comments

Comments
 (0)