Skip to content

Commit 4e7666a

Browse files
morgandumorgandu
authored
Feat: add labels to all resource creation apis (#601)
Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [x] Ensure the tests and linter pass - [x] Code coverage does not decrease (if any source code was changed) - [x] Appropriate docs were updated (if necessary) Fixes #595, <b/191789899> 🦕 - Add support for labels in resources creation: - datasets (`create`) - ImageDataset / TabularDataset / TextDataset / TimeSeriesDataset / VideoDataset - tensorboard - `Tensorboard.create` - jobs - `BatchPredictionJob.create` - `CustomJob.__init__` - `HyperparameterTuningJob.__init__` - models - `Endpoint.create` - `Model.upload` / `Model.batch_predict` - training_jobs (`__init__` for training labels, `run` for model labels) - CustomTrainingJob / CustomContainerTrainingJob / CustomPythonPackageTrainingJob - AutoMLTabularTrainingJob / AutoMLForecastingTrainingJob / AutoMLImageTrainingJob / AutoMLVideoTrainingJob / AutoMLTextTrainingJob - Modify `pipeline_jobs.py` and `utils.py` for `validate_labels` reusability - Add / modify unit tests to verify: - standard resource creation with / without labels - training pipeline resource creation - when model labels is not provided - when base model labels is provided
1 parent b478075 commit 4e7666a

24 files changed

+872
-64
lines changed

‎google/cloud/aiplatform/datasets/dataset.py‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# limitations under the License.
1616
#
1717

18-
from typing import Optional, Sequence, Dict, Tuple, Union, List
18+
from typing import Dict, List, Optional, Sequence, Tuple, Union
1919

2020
from google.api_core import operation
2121
from google.auth import credentials as auth_credentials
@@ -115,6 +115,7 @@ def create(
115115
location: Optional[str] = None,
116116
credentials: Optional[auth_credentials.Credentials] = None,
117117
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
118+
labels: Optional[Dict[str, str]] = None,
118119
encryption_spec_key_name: Optional[str] = None,
119120
sync: bool = True,
120121
) -> "_Dataset":
@@ -176,6 +177,16 @@ def create(
176177
credentials set in aiplatform.init.
177178
request_metadata (Sequence[Tuple[str, str]]):
178179
Strings which should be sent along with the request as metadata.
180+
labels (Dict[str, str]):
181+
Optional. Labels with user-defined metadata to organize your Tensorboards.
182+
Label keys and values can be no longer than 64 characters
183+
(Unicode codepoints), can only contain lowercase letters, numeric
184+
characters, underscores and dashes. International characters are allowed.
185+
No more than 64 user labels can be associated with one Tensorboard
186+
(System labels are excluded).
187+
See https://goo.gl/xmQnxf for more information and examples of labels.
188+
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
189+
and are immutable.
179190
encryption_spec_key_name (Optional[str]):
180191
Optional. The Cloud KMS resource identifier of the customer
181192
managed encryption key used to protect the dataset. Has the
@@ -198,6 +209,8 @@ def create(
198209
"""
199210

200211
utils.validate_display_name(display_name)
212+
if labels:
213+
utils.validate_labels(labels)
201214

202215
api_client = cls._instantiate_client(location=location, credentials=credentials)
203216

@@ -221,6 +234,7 @@ def create(
221234
location=location or initializer.global_config.location,
222235
credentials=credentials or initializer.global_config.credentials,
223236
request_metadata=request_metadata,
237+
labels=labels,
224238
encryption_spec=initializer.global_config.get_encryption_spec(
225239
encryption_spec_key_name=encryption_spec_key_name
226240
),
@@ -240,6 +254,7 @@ def _create_and_import(
240254
location: str,
241255
credentials: Optional[auth_credentials.Credentials],
242256
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
257+
labels: Optional[Dict[str, str]] = None,
243258
encryption_spec: Optional[gca_encryption_spec.EncryptionSpec] = None,
244259
sync: bool = True,
245260
) -> "_Dataset":
@@ -277,6 +292,16 @@ def _create_and_import(
277292
credentials set in aiplatform.init.
278293
request_metadata (Sequence[Tuple[str, str]]):
279294
Strings which should be sent along with the request as metadata.
295+
labels (Dict[str, str]):
296+
Optional. Labels with user-defined metadata to organize your Tensorboards.
297+
Label keys and values can be no longer than 64 characters
298+
(Unicode codepoints), can only contain lowercase letters, numeric
299+
characters, underscores and dashes. International characters are allowed.
300+
No more than 64 user labels can be associated with one Tensorboard
301+
(System labels are excluded).
302+
See https://goo.gl/xmQnxf for more information and examples of labels.
303+
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
304+
and are immutable.
280305
encryption_spec (Optional[gca_encryption_spec.EncryptionSpec]):
281306
Optional. The Cloud KMS customer managed encryption key used to protect the dataset.
282307
The key needs to be in the same region as where the compute
@@ -300,6 +325,7 @@ def _create_and_import(
300325
metadata_schema_uri=metadata_schema_uri,
301326
datasource=datasource,
302327
request_metadata=request_metadata,
328+
labels=labels,
303329
encryption_spec=encryption_spec,
304330
)
305331

@@ -346,6 +372,7 @@ def _create(
346372
metadata_schema_uri: str,
347373
datasource: _datasources.Datasource,
348374
request_metadata: Sequence[Tuple[str, str]] = (),
375+
labels: Optional[Dict[str, str]] = None,
349376
encryption_spec: Optional[gca_encryption_spec.EncryptionSpec] = None,
350377
) -> operation.Operation:
351378
"""Creates a new managed dataset by directly calling API client.
@@ -373,6 +400,16 @@ def _create(
373400
request_metadata (Sequence[Tuple[str, str]]):
374401
Strings which should be sent along with the create_dataset
375402
request as metadata. Usually to specify special dataset config.
403+
labels (Dict[str, str]):
404+
Optional. Labels with user-defined metadata to organize your Tensorboards.
405+
Label keys and values can be no longer than 64 characters
406+
(Unicode codepoints), can only contain lowercase letters, numeric
407+
characters, underscores and dashes. International characters are allowed.
408+
No more than 64 user labels can be associated with one Tensorboard
409+
(System labels are excluded).
410+
See https://goo.gl/xmQnxf for more information and examples of labels.
411+
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
412+
and are immutable.
376413
encryption_spec (Optional[gca_encryption_spec.EncryptionSpec]):
377414
Optional. The Cloud KMS customer managed encryption key used to protect the dataset.
378415
The key needs to be in the same region as where the compute
@@ -388,6 +425,7 @@ def _create(
388425
display_name=display_name,
389426
metadata_schema_uri=metadata_schema_uri,
390427
metadata=datasource.dataset_metadata,
428+
labels=labels,
391429
encryption_spec=encryption_spec,
392430
)
393431

‎google/cloud/aiplatform/datasets/image_dataset.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# limitations under the License.
1616
#
1717

18-
from typing import Optional, Sequence, Dict, Tuple, Union
18+
from typing import Dict, Optional, Sequence, Tuple, Union
1919

2020
from google.auth import credentials as auth_credentials
2121

@@ -44,6 +44,7 @@ def create(
4444
location: Optional[str] = None,
4545
credentials: Optional[auth_credentials.Credentials] = None,
4646
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
47+
labels: Optional[Dict[str, str]] = None,
4748
encryption_spec_key_name: Optional[str] = None,
4849
sync: bool = True,
4950
) -> "ImageDataset":
@@ -95,6 +96,16 @@ def create(
9596
credentials set in aiplatform.init.
9697
request_metadata (Sequence[Tuple[str, str]]):
9798
Strings which should be sent along with the request as metadata.
99+
labels (Dict[str, str]):
100+
Optional. Labels with user-defined metadata to organize your Tensorboards.
101+
Label keys and values can be no longer than 64 characters
102+
(Unicode codepoints), can only contain lowercase letters, numeric
103+
characters, underscores and dashes. International characters are allowed.
104+
No more than 64 user labels can be associated with one Tensorboard
105+
(System labels are excluded).
106+
See https://goo.gl/xmQnxf for more information and examples of labels.
107+
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
108+
and are immutable.
98109
encryption_spec_key_name (Optional[str]):
99110
Optional. The Cloud KMS resource identifier of the customer
100111
managed encryption key used to protect the dataset. Has the
@@ -117,6 +128,8 @@ def create(
117128
"""
118129

119130
utils.validate_display_name(display_name)
131+
if labels:
132+
utils.validate_labels(labels)
120133

121134
api_client = cls._instantiate_client(location=location, credentials=credentials)
122135

@@ -141,6 +154,7 @@ def create(
141154
location=location or initializer.global_config.location,
142155
credentials=credentials or initializer.global_config.credentials,
143156
request_metadata=request_metadata,
157+
labels=labels,
144158
encryption_spec=initializer.global_config.get_encryption_spec(
145159
encryption_spec_key_name=encryption_spec_key_name
146160
),

‎google/cloud/aiplatform/datasets/tabular_dataset.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import csv
1919
import logging
2020

21-
from typing import List, Optional, Sequence, Set, Tuple, Union
21+
from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
2222

2323
from google.auth import credentials as auth_credentials
2424

@@ -269,6 +269,7 @@ def create(
269269
location: Optional[str] = None,
270270
credentials: Optional[auth_credentials.Credentials] = None,
271271
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
272+
labels: Optional[Dict[str, str]] = None,
272273
encryption_spec_key_name: Optional[str] = None,
273274
sync: bool = True,
274275
) -> "TabularDataset":
@@ -302,6 +303,16 @@ def create(
302303
credentials set in aiplatform.init.
303304
request_metadata (Sequence[Tuple[str, str]]):
304305
Strings which should be sent along with the request as metadata.
306+
labels (Dict[str, str]):
307+
Optional. Labels with user-defined metadata to organize your Tensorboards.
308+
Label keys and values can be no longer than 64 characters
309+
(Unicode codepoints), can only contain lowercase letters, numeric
310+
characters, underscores and dashes. International characters are allowed.
311+
No more than 64 user labels can be associated with one Tensorboard
312+
(System labels are excluded).
313+
See https://goo.gl/xmQnxf for more information and examples of labels.
314+
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
315+
and are immutable.
305316
encryption_spec_key_name (Optional[str]):
306317
Optional. The Cloud KMS resource identifier of the customer
307318
managed encryption key used to protect the dataset. Has the
@@ -324,6 +335,8 @@ def create(
324335
"""
325336

326337
utils.validate_display_name(display_name)
338+
if labels:
339+
utils.validate_labels(labels)
327340

328341
api_client = cls._instantiate_client(location=location, credentials=credentials)
329342

@@ -347,6 +360,7 @@ def create(
347360
location=location or initializer.global_config.location,
348361
credentials=credentials or initializer.global_config.credentials,
349362
request_metadata=request_metadata,
363+
labels=labels,
350364
encryption_spec=initializer.global_config.get_encryption_spec(
351365
encryption_spec_key_name=encryption_spec_key_name
352366
),

‎google/cloud/aiplatform/datasets/text_dataset.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# limitations under the License.
1616
#
1717

18-
from typing import Optional, Sequence, Dict, Tuple, Union
18+
from typing import Dict, Optional, Sequence, Tuple, Union
1919

2020
from google.auth import credentials as auth_credentials
2121

@@ -44,6 +44,7 @@ def create(
4444
location: Optional[str] = None,
4545
credentials: Optional[auth_credentials.Credentials] = None,
4646
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
47+
labels: Optional[Dict[str, str]] = None,
4748
encryption_spec_key_name: Optional[str] = None,
4849
sync: bool = True,
4950
) -> "TextDataset":
@@ -102,6 +103,16 @@ def create(
102103
credentials set in aiplatform.init.
103104
request_metadata (Sequence[Tuple[str, str]]):
104105
Strings which should be sent along with the request as metadata.
106+
labels (Dict[str, str]):
107+
Optional. Labels with user-defined metadata to organize your Tensorboards.
108+
Label keys and values can be no longer than 64 characters
109+
(Unicode codepoints), can only contain lowercase letters, numeric
110+
characters, underscores and dashes. International characters are allowed.
111+
No more than 64 user labels can be associated with one Tensorboard
112+
(System labels are excluded).
113+
See https://goo.gl/xmQnxf for more information and examples of labels.
114+
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
115+
and are immutable.
105116
encryption_spec_key_name (Optional[str]):
106117
Optional. The Cloud KMS resource identifier of the customer
107118
managed encryption key used to protect the dataset. Has the
@@ -124,6 +135,8 @@ def create(
124135
"""
125136

126137
utils.validate_display_name(display_name)
138+
if labels:
139+
utils.validate_labels(labels)
127140

128141
api_client = cls._instantiate_client(location=location, credentials=credentials)
129142

@@ -148,6 +161,7 @@ def create(
148161
location=location or initializer.global_config.location,
149162
credentials=credentials or initializer.global_config.credentials,
150163
request_metadata=request_metadata,
164+
labels=labels,
151165
encryption_spec=initializer.global_config.get_encryption_spec(
152166
encryption_spec_key_name=encryption_spec_key_name
153167
),

‎google/cloud/aiplatform/datasets/time_series_dataset.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# limitations under the License.
1616
#
1717

18-
from typing import Optional, Sequence, Tuple, Union
18+
from typing import Dict, Optional, Sequence, Tuple, Union
1919

2020
from google.auth import credentials as auth_credentials
2121

@@ -43,6 +43,7 @@ def create(
4343
location: Optional[str] = None,
4444
credentials: Optional[auth_credentials.Credentials] = None,
4545
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
46+
labels: Optional[Dict[str, str]] = None,
4647
encryption_spec_key_name: Optional[str] = None,
4748
sync: bool = True,
4849
) -> "TimeSeriesDataset":
@@ -76,6 +77,16 @@ def create(
7677
credentials set in aiplatform.init.
7778
request_metadata (Sequence[Tuple[str, str]]):
7879
Strings which should be sent along with the request as metadata.
80+
labels (Dict[str, str]):
81+
Optional. Labels with user-defined metadata to organize your Tensorboards.
82+
Label keys and values can be no longer than 64 characters
83+
(Unicode codepoints), can only contain lowercase letters, numeric
84+
characters, underscores and dashes. International characters are allowed.
85+
No more than 64 user labels can be associated with one Tensorboard
86+
(System labels are excluded).
87+
See https://goo.gl/xmQnxf for more information and examples of labels.
88+
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
89+
and are immutable.
7990
encryption_spec_key_name (Optional[str]):
8091
Optional. The Cloud KMS resource identifier of the customer
8192
managed encryption key used to protect the dataset. Has the
@@ -99,6 +110,8 @@ def create(
99110
"""
100111

101112
utils.validate_display_name(display_name)
113+
if labels:
114+
utils.validate_labels(labels)
102115

103116
api_client = cls._instantiate_client(location=location, credentials=credentials)
104117

@@ -122,6 +135,7 @@ def create(
122135
location=location or initializer.global_config.location,
123136
credentials=credentials or initializer.global_config.credentials,
124137
request_metadata=request_metadata,
138+
labels=labels,
125139
encryption_spec=initializer.global_config.get_encryption_spec(
126140
encryption_spec_key_name=encryption_spec_key_name
127141
),

‎google/cloud/aiplatform/datasets/video_dataset.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# limitations under the License.
1616
#
1717

18-
from typing import Optional, Sequence, Dict, Tuple, Union
18+
from typing import Dict, Optional, Sequence, Tuple, Union
1919

2020
from google.auth import credentials as auth_credentials
2121

@@ -44,6 +44,7 @@ def create(
4444
location: Optional[str] = None,
4545
credentials: Optional[auth_credentials.Credentials] = None,
4646
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
47+
labels: Optional[Dict[str, str]] = None,
4748
encryption_spec_key_name: Optional[str] = None,
4849
sync: bool = True,
4950
) -> "VideoDataset":
@@ -95,6 +96,16 @@ def create(
9596
credentials set in aiplatform.init.
9697
request_metadata (Sequence[Tuple[str, str]]):
9798
Strings which should be sent along with the request as metadata.
99+
labels (Dict[str, str]):
100+
Optional. Labels with user-defined metadata to organize your Tensorboards.
101+
Label keys and values can be no longer than 64 characters
102+
(Unicode codepoints), can only contain lowercase letters, numeric
103+
characters, underscores and dashes. International characters are allowed.
104+
No more than 64 user labels can be associated with one Tensorboard
105+
(System labels are excluded).
106+
See https://goo.gl/xmQnxf for more information and examples of labels.
107+
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
108+
and are immutable.
98109
encryption_spec_key_name (Optional[str]):
99110
Optional. The Cloud KMS resource identifier of the customer
100111
managed encryption key used to protect the dataset. Has the
@@ -117,6 +128,8 @@ def create(
117128
"""
118129

119130
utils.validate_display_name(display_name)
131+
if labels:
132+
utils.validate_labels(labels)
120133

121134
api_client = cls._instantiate_client(location=location, credentials=credentials)
122135

@@ -141,6 +154,7 @@ def create(
141154
location=location or initializer.global_config.location,
142155
credentials=credentials or initializer.global_config.credentials,
143156
request_metadata=request_metadata,
157+
labels=labels,
144158
encryption_spec=initializer.global_config.get_encryption_spec(
145159
encryption_spec_key_name=encryption_spec_key_name
146160
),

0 commit comments

Comments
 (0)