From 0cb1ee4000f62de623a4cde5d96b77480ac8d35a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 10:17:51 -0400 Subject: [PATCH 01/69] feat: separate pr description generation from entrypoint --- library_generation/cli/entry_point.py | 8 -- .../cli/generate_pr_description.py | 89 +++++++++++++++++++ 2 files changed, 89 insertions(+), 8 deletions(-) create mode 100644 library_generation/cli/generate_pr_description.py diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 5ca6d93219..9883ce6b61 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -15,7 +15,6 @@ import sys from typing import Optional import click as click -from library_generation.generate_pr_description import generate_pr_descriptions from library_generation.generate_repo import generate_from_yaml from library_generation.model.config_change import ConfigChange from library_generation.model.generation_config import from_yaml @@ -116,9 +115,6 @@ def generate( specified. Raise FileNotFoundError if the default config does not exist. - - The commit history, if generated, will be available in - repository_path/pr_description.txt. """ __generate_repo_and_pr_description_impl( baseline_generation_config_path=baseline_generation_config_path, @@ -206,10 +202,6 @@ def __generate_repo_and_pr_description_impl( api_definitions_path=api_definitions_path, target_library_names=target_library_names, ) - generate_pr_descriptions( - config_change=config_change, - description_path=repository_path, - ) def _needs_full_repo_generation(config_change: ConfigChange) -> bool: diff --git a/library_generation/cli/generate_pr_description.py b/library_generation/cli/generate_pr_description.py new file mode 100644 index 0000000000..3f66bf6f42 --- /dev/null +++ b/library_generation/cli/generate_pr_description.py @@ -0,0 +1,89 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import click as click + +from library_generation.generate_pr_description import generate_pr_descriptions +from library_generation.model.generation_config import from_yaml +from library_generation.utils.generation_config_comparator import compare_config + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--baseline-generation-config-path", + required=False, + default=None, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml. + This config file is used for commit history generation, not library + generation. + """, +) +@click.option( + "--current-generation-config-path", + required=False, + default=None, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml that contains the + metadata about library generation. + """, +) +@click.option( + "--repository-path", + type=str, + default=".", + show_default=True, + help=""" + The repository path to which the generated files + will be sent. + If not specified, the repository will be generated to the current working + directory. + """, +) +def generate( + baseline_generation_config_path: str, + current_generation_config_path: str, + repository_path: str, +) -> None: + baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) + current_generation_config_path = os.path.abspath(current_generation_config_path) + if not ( + os.path.isfile(baseline_generation_config_path) + and os.path.isfile(current_generation_config_path) + ): + # One of the generation configs does not exist, do not generate + # the description. + return + config_change = compare_config( + baseline_config=from_yaml(baseline_generation_config_path), + current_config=from_yaml(current_generation_config_path), + ) + generate_pr_descriptions( + config_change=config_change, + description_path=repository_path, + ) + + +if __name__ == "__main__": + main() From 7b9e493da1d61060a08e1dd74622cc15bfa16e28 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 10:47:42 -0400 Subject: [PATCH 02/69] add unit test --- library_generation/cli/entry_point.py | 7 +- ...escription.py => generate_release_note.py} | 25 ++++--- .../cli/generate_release_note_unit_tests.py | 71 +++++++++++++++++++ .../empty_gen_config.yaml | 0 4 files changed, 90 insertions(+), 13 deletions(-) rename library_generation/cli/{generate_pr_description.py => generate_release_note.py} (80%) create mode 100644 library_generation/test/cli/generate_release_note_unit_tests.py create mode 100644 library_generation/test/resources/test_generate_release_note/empty_gen_config.yaml diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 9883ce6b61..a41d25e500 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -93,12 +93,11 @@ def generate( ): """ Compare baseline generation config and current generation config and - generate changed libraries based on current generation config with commit - history. + generate changed libraries based on current generation config. If baseline generation config is not specified but current generation config is specified, generate all libraries if `library_names` is not - specified, based on current generation config without commit history. + specified, based on current generation config. If current generation config is not specified but baseline generation config is specified, raise FileNotFoundError because current generation @@ -168,7 +167,6 @@ def __generate_repo_and_pr_description_impl( if not baseline_generation_config_path: # Execute selective generation based on current_generation_config if # baseline_generation_config is not specified. - # Do not generate pull request description. generate_from_yaml( config=from_yaml(current_generation_config_path), repository_path=repository_path, @@ -178,7 +176,6 @@ def __generate_repo_and_pr_description_impl( return # Compare two generation configs to get changed libraries. - # Generate pull request description. baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) config_change = compare_config( baseline_config=from_yaml(baseline_generation_config_path), diff --git a/library_generation/cli/generate_pr_description.py b/library_generation/cli/generate_release_note.py similarity index 80% rename from library_generation/cli/generate_pr_description.py rename to library_generation/cli/generate_release_note.py index 3f66bf6f42..ce8ca9e7ac 100644 --- a/library_generation/cli/generate_pr_description.py +++ b/library_generation/cli/generate_release_note.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import os - +from typing import Optional import click as click - from library_generation.generate_pr_description import generate_pr_descriptions from library_generation.model.generation_config import from_yaml from library_generation.utils.generation_config_comparator import compare_config @@ -55,25 +54,35 @@ def main(ctx): default=".", show_default=True, help=""" - The repository path to which the generated files - will be sent. + The repository path to which the generated files will be sent. If not specified, the repository will be generated to the current working directory. """, ) def generate( - baseline_generation_config_path: str, - current_generation_config_path: str, + baseline_generation_config_path: Optional[str], + current_generation_config_path: Optional[str], repository_path: str, ) -> None: + if ( + baseline_generation_config_path is None + or current_generation_config_path is None + ): + print( + "One of the generation configs is not specified, do not generate " + "the description." + ) + return baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) current_generation_config_path = os.path.abspath(current_generation_config_path) if not ( os.path.isfile(baseline_generation_config_path) and os.path.isfile(current_generation_config_path) ): - # One of the generation configs does not exist, do not generate - # the description. + print( + "One of the generation configs does not exist, do not generate " + "the description." + ) return config_change = compare_config( baseline_config=from_yaml(baseline_generation_config_path), diff --git a/library_generation/test/cli/generate_release_note_unit_tests.py b/library_generation/test/cli/generate_release_note_unit_tests.py new file mode 100644 index 0000000000..be0d6a4fed --- /dev/null +++ b/library_generation/test/cli/generate_release_note_unit_tests.py @@ -0,0 +1,71 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +from click.testing import CliRunner +from library_generation.cli.generate_release_note import generate + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resource_dir = os.path.join(script_dir, "..", "resources", "test_generate_release_note") + + +class GenerateReleaseNoteTest(unittest.TestCase): + def test_gen_release_note_with_no_baseline_config_does_not_generate_note(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke(generate) + self.assertEqual(0, result.exit_code) + self.assertFalse(os.path.isfile("pr_description.txt")) + + def test_gen_release_note_with_no_current_config_does_not_generate_note(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + generate, ["--baseline-generation-config-path=any_config"] + ) + self.assertEqual(0, result.exit_code) + self.assertFalse(os.path.isfile("pr_description.txt")) + + def test_gen_release_note_with_nonexistent_baseline_config_does_not_generate_note( + self, + ): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + generate, + [ + "--baseline-generation-config-path=non_existent_config", + "--current-generation-config-path=not_relevant", + ], + ) + self.assertEqual(0, result.exit_code) + self.assertFalse(os.path.isfile("pr_description.txt")) + + def test_gen_release_note_with_nonexistent_current_config_does_not_generate_note( + self, + ): + cwd = os.getcwd() + os.chdir(resource_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + generate, + [ + "--baseline-generation-config-path=empty_gen_config.yaml", + "--current-generation-config-path=non_existent_config", + ], + ) + self.assertEqual(0, result.exit_code) + self.assertFalse(os.path.isfile("pr_description.txt")) + os.chdir(cwd) diff --git a/library_generation/test/resources/test_generate_release_note/empty_gen_config.yaml b/library_generation/test/resources/test_generate_release_note/empty_gen_config.yaml new file mode 100644 index 0000000000..e69de29bb2 From f74d1c540be35ccf42332601db42a3e053857ffd Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 10:58:03 -0400 Subject: [PATCH 03/69] fix unit test --- .../test/cli/entry_point_unit_tests.py | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py index ca70bdd647..80197ab2df 100644 --- a/library_generation/test/cli/entry_point_unit_tests.py +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -86,10 +86,8 @@ def test_validate_generation_config_with_duplicate_library_name_raise_file_excep ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_non_monorepo_without_changes_triggers_full_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ @@ -116,10 +114,8 @@ def test_generate_non_monorepo_without_changes_triggers_full_generation( ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_non_monorepo_without_changes_with_includes_triggers_selective_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ @@ -147,10 +143,8 @@ def test_generate_non_monorepo_without_changes_with_includes_triggers_selective_ ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_non_monorepo_with_changes_triggers_full_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ @@ -181,10 +175,8 @@ def test_generate_non_monorepo_with_changes_triggers_full_generation( ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_non_monorepo_with_changes_with_includes_triggers_selective_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ @@ -216,10 +208,8 @@ def test_generate_non_monorepo_with_changes_with_includes_triggers_selective_gen ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_monorepo_with_common_protos_triggers_full_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ @@ -247,10 +237,8 @@ def test_generate_monorepo_with_common_protos_triggers_full_generation( ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ @@ -277,10 +265,8 @@ def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_g ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_monorepo_without_change_does_not_trigger_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ @@ -309,10 +295,8 @@ def test_generate_monorepo_without_change_does_not_trigger_generation( ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_monorepo_without_change_with_includes_trigger_selective_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ @@ -341,10 +325,8 @@ def test_generate_monorepo_without_change_with_includes_trigger_selective_genera ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_monorepo_with_changed_config_without_includes_trigger_changed_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ @@ -374,10 +356,8 @@ def test_generate_monorepo_with_changed_config_without_includes_trigger_changed_ ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_monorepo_with_changed_config_and_includes_trigger_selective_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ @@ -407,10 +387,8 @@ def test_generate_monorepo_with_changed_config_and_includes_trigger_selective_ge ) @patch("library_generation.cli.entry_point.generate_from_yaml") - @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_monorepo_without_changed_config_without_includes_does_not_trigger_generation( self, - generate_pr_descriptions, generate_from_yaml, ): """ From 99eb34a9752d486549f6a3ae8c5ae5a096c17239 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 11:18:48 -0400 Subject: [PATCH 04/69] fix integration test --- library_generation/test/integration_tests.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index c5f8221cc9..7a21a02704 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from click.testing import CliRunner import difflib import json import tempfile @@ -24,11 +25,15 @@ from distutils.dir_util import copy_tree from distutils.file_util import copy_file from pathlib import Path +from library_generation.cli.generate_release_note import ( + generate as generate_pr_description, +) from library_generation.model.generation_config import GenerationConfig from library_generation.model.generation_config import from_yaml from library_generation.test.compare_poms import compare_xml from library_generation.utils.utilities import sh_util as shell_call + script_dir = os.path.dirname(os.path.realpath(__file__)) config_dir = os.path.join(script_dir, "resources", "integration") golden_dir = os.path.join(config_dir, "golden") @@ -94,7 +99,18 @@ def test_entry_point_running_in_container(self): current_config=current_config_name, api_definition=api_definitions_path, ) - # 4. compare generation result with golden files + # 4. generate pr description + # noinspection PyTypeChecker + result = CliRunner().invoke( + generate_pr_description, + [ + f"--baseline-generation-config-path={baseline_config_name}", + f"--current-generation-config-path={current_config_name}", + f"--repository-path={repo_location}", + ], + ) + self.assertEqual(0, result.exit_code) + # 5. compare generation result with golden files print( "Generation finished successfully. " "Will now compare differences between generated and existing " From c87d6fb54b254a4cacf2fb5ddefe7c006da1fa71 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 11:33:46 -0400 Subject: [PATCH 05/69] fix integration test --- library_generation/test/integration_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 7a21a02704..576fb21ede 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -104,8 +104,8 @@ def test_entry_point_running_in_container(self): result = CliRunner().invoke( generate_pr_description, [ - f"--baseline-generation-config-path={baseline_config_name}", - f"--current-generation-config-path={current_config_name}", + f"--baseline-generation-config-path={config_location}/{baseline_config_name}", + f"--current-generation-config-path={config_location}/{current_config_name}", f"--repository-path={repo_location}", ], ) From c5dfb844cdf07517f2065d08b1750587485e6bee Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 12:00:02 -0400 Subject: [PATCH 06/69] change shell script --- .github/scripts/hermetic_library_generation.sh | 11 +++++++++++ library_generation/cli/generate_release_note.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 2c5552f77d..0d757c96d3 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -105,6 +105,17 @@ docker run \ --current-generation-config-path="${workspace_name}/${generation_config}" \ --api-definitions-path="${workspace_name}/googleapis" +# generate pr description +docker run \ + --rm \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):${workspace_name}" \ + --entrypoint python /src/cli/generate_release_note.py generate \ + gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ + --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ + --current-generation-config-path="${workspace_name}/${generation_config}" \ + --repository-path="${workspace_name}" + # remove api definitions after generation rm -rf "${api_def_dir}" diff --git a/library_generation/cli/generate_release_note.py b/library_generation/cli/generate_release_note.py index ce8ca9e7ac..097496e9a4 100644 --- a/library_generation/cli/generate_release_note.py +++ b/library_generation/cli/generate_release_note.py @@ -90,7 +90,7 @@ def generate( ) generate_pr_descriptions( config_change=config_change, - description_path=repository_path, + description_path=os.path.abspath(repository_path), ) From 758f0bc25530e4ae854b38e75f143e6e34111fcd Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 12:15:44 -0400 Subject: [PATCH 07/69] change entrypoint --- .github/scripts/hermetic_library_generation.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 0d757c96d3..071ee47a0a 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -110,8 +110,9 @@ docker run \ --rm \ -u "$(id -u):$(id -g)" \ -v "$(pwd):${workspace_name}" \ - --entrypoint python /src/cli/generate_release_note.py generate \ + --entrypoint python \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ + /src/cli/generate_release_note.py generate \ --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ --current-generation-config-path="${workspace_name}/${generation_config}" \ --repository-path="${workspace_name}" From 8fa281e9b5884c909d8de78630525a234eaea2d7 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 12:26:02 -0400 Subject: [PATCH 08/69] add env --- .github/scripts/hermetic_library_generation.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 071ee47a0a..6a6d0f4d99 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -108,8 +108,10 @@ docker run \ # generate pr description docker run \ --rm \ + --quiet \ -u "$(id -u):$(id -g)" \ -v "$(pwd):${workspace_name}" \ + -e GENERATOR_VERSION="${image_tag}" \ --entrypoint python \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ /src/cli/generate_release_note.py generate \ From 8ae1360a48e5469701bbaa019953baf21728a443 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 13:20:04 -0400 Subject: [PATCH 09/69] change doc --- .../library_generation.Dockerfile | 2 +- library_generation/README.md | 16 ++++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 41eb1c1a33..3131028252 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -79,7 +79,7 @@ RUN python -m pip install --upgrade pip # install main scripts as a python package WORKDIR /src -RUN python -m pip install -r requirements.txt +RUN python -m pip install --require-hashes -r requirements.txt RUN python -m pip install . # Install nvm with node and npm diff --git a/library_generation/README.md b/library_generation/README.md index 20a698a6ab..518d588f9a 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -1,8 +1,8 @@ # Generate a repository containing GAPIC Client Libraries The script, `entry_point.py`, allows you to generate a repository containing -GAPIC client libraries with change history (a monorepo, for example, -google-cloud-java) from a configuration file. +GAPIC client libraries (a monorepo, for example, google-cloud-java) from a +configuration file. ## Environment @@ -23,7 +23,7 @@ Please refer to [Repository path](#repository-path--repositorypath---optional) f ### Baseline generation configuration yaml (`baseline_generation_config`) An absolute or relative path to a generation_config.yaml. -This config file is used for commit history generation, not library +This config file is used for computing changed libraries, not library generation. ### Current generation configuration yaml (`current_generation_config`) @@ -89,14 +89,6 @@ will be created/modified: | pom.xml (repo root dir) | Always generated from inputs | | versions.txt | New entries will be added if they don’t exist | -### Change history - -If both `baseline_generation_config` and `current_generation_config` are -specified and the contents are different, the changed contents will be generated -into `pr_description.txt` in the `repository_path`. -In addition, if the `googleapis_commitish` is different, the googleapis commit -history will be generated. - ## Configuration to generate a repository There are three levels of parameters in the configuration: repository level, @@ -201,7 +193,7 @@ The virtual environment can be installed to any folder, usually it is recommende Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation` ```bash - python -m pip install -r library_generation/requirements.txt + python -m pip install --require-hashes -r library_generation/requirements.txt ``` 3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere From 19de9cdd17c026396ea0f7c01d038d8fec1b0fcb Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 13:20:43 -0400 Subject: [PATCH 10/69] change ci --- .github/workflows/verify_library_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index 9682f8e594..c4eb297126 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -86,7 +86,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.txt + pip install --require-hashes -r requirements.txt pip install . popd - name: Run shell unit tests From ff85b71708da2b447ae1af439dfec7f9b54004e9 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 17 Oct 2024 13:22:52 -0400 Subject: [PATCH 11/69] change ci --- .github/workflows/verify_library_generation.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index c4eb297126..722ea7db95 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -64,7 +64,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.txt + pip install --require-hashes -r requirements.txt pip install . popd - name: Run integration tests @@ -120,7 +120,7 @@ jobs: run: | set -ex pushd library_generation - pip install -r requirements.txt + pip install --require-hashes -r requirements.txt popd - name: Lint shell: bash From 72b3acb6b8c802edc36621a42ef6f6ea980ab0e7 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 08:27:50 -0400 Subject: [PATCH 12/69] create separate packages --- hermetic_build/common/__init__.py | 0 hermetic_build/library_generation/__init__.py | 0 hermetic_build/release_note_generation/__init__.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 hermetic_build/common/__init__.py create mode 100644 hermetic_build/library_generation/__init__.py create mode 100644 hermetic_build/release_note_generation/__init__.py diff --git a/hermetic_build/common/__init__.py b/hermetic_build/common/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/__init__.py b/hermetic_build/library_generation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/release_note_generation/__init__.py b/hermetic_build/release_note_generation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 3c7e8f058d029ca34e0af1f2358c29df75f73325 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 09:10:37 -0400 Subject: [PATCH 13/69] setup common project --- hermetic_build/common/model/gapic_config.py | 99 +++++++ .../common/model/generation_config.py | 198 +++++++++++++ hermetic_build/common/model/library_config.py | 203 ++++++++++++++ hermetic_build/common/setup.py | 17 ++ .../tests/model/gapic_config_unit_tests.py | 122 +++++++++ .../model/generation_config_unit_tests.py | 259 ++++++++++++++++++ .../tests/model/library_config_unit_tests.py | 122 +++++++++ .../config_without_api_description.yaml | 5 + .../config_without_api_shortname.yaml | 4 + .../config_without_gapics_key.yaml | 3 + .../config_without_gapics_value.yaml | 4 + .../config_without_googleapis.yaml | 8 + .../test-config/config_without_libraries.yaml | 1 + .../config_without_library_value.yaml | 2 + .../config_without_name_pretty.yaml | 6 + .../config_without_product_docs.yaml | 7 + .../config_without_proto_path.yaml | 4 + .../config_without_temp_excludes.yaml | 10 + .../test-config/generation_config.yaml | 24 ++ .../generation_config_library_modified.yaml | 14 + ...on_config_with_duplicate_library_name.yaml | 51 ++++ .../test-config/monorepo_baseline.yaml | 30 ++ .../test-config/monorepo_current.yaml | 30 ++ .../monorepo_with_common_protos.yaml | 43 +++ .../monorepo_without_common_protos.yaml | 33 +++ 25 files changed, 1299 insertions(+) create mode 100644 hermetic_build/common/model/gapic_config.py create mode 100644 hermetic_build/common/model/generation_config.py create mode 100644 hermetic_build/common/model/library_config.py create mode 100755 hermetic_build/common/setup.py create mode 100644 hermetic_build/common/tests/model/gapic_config_unit_tests.py create mode 100644 hermetic_build/common/tests/model/generation_config_unit_tests.py create mode 100644 hermetic_build/common/tests/model/library_config_unit_tests.py create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_api_description.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_api_shortname.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_gapics_key.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_gapics_value.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_googleapis.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_libraries.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_library_value.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_name_pretty.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_product_docs.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_proto_path.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/config_without_temp_excludes.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/generation_config.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/generation_config_library_modified.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/monorepo_baseline.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/monorepo_current.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/monorepo_with_common_protos.yaml create mode 100644 hermetic_build/common/tests/resources/test-config/monorepo_without_common_protos.yaml diff --git a/hermetic_build/common/model/gapic_config.py b/hermetic_build/common/model/gapic_config.py new file mode 100644 index 0000000000..3bbba39454 --- /dev/null +++ b/hermetic_build/common/model/gapic_config.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re +from typing import Optional + +ALPHA_VERSION = "alpha" +BETA_VERSION = "beta" + + +class GapicConfig: + """ + Class that represents a GAPICs single entry, inside a `LibraryConfig` in + a generation_config.yaml + """ + + def __init__(self, proto_path: str): + self.proto_path = proto_path + self.version = self.__parse_version() + + def is_stable(self): + return ( + self.version + and (ALPHA_VERSION not in self.version) + and (BETA_VERSION not in self.version) + ) + + def get_version(self): + return self.version + + def __parse_version(self) -> Optional[str]: + version_regex = re.compile(r"^v[1-9]+(p[1-9]+)*(alpha|beta)?.*") + for directory in self.proto_path.split("/"): + if version_regex.search(directory): + return directory + return None + + def __eq__(self, other) -> bool: + if not isinstance(other, GapicConfig): + return False + return self.proto_path == other.proto_path + + def __lt__(self, other) -> bool: + if not isinstance(other, GapicConfig): + raise ValueError( + f"Type {type(other)} can't be comparable " f"with GapicConfig." + ) + + self_version = self.get_version() + other_version = other.get_version() + self_dirs = self.proto_path.split("/") + other_dirs = other.proto_path.split("/") + # Case 1: if both of the configs don't have a version in proto_path, + # the one with lower depth is smaller. + if (not self_version) and (not other_version): + return len(self_dirs) < len(other_dirs) + # Case 2: if only one config has a version in proto_path, it is smaller + # than the other one. + if self_version and (not other_version): + return True + if (not self_version) and other_version: + return False + # Two configs both have a version in proto_path. + self_stable = self.is_stable() + other_stable = other.is_stable() + # Case 3, if only config has a stable version in proto_path, it is + # smaller than the other one. + if self_stable and (not other_stable): + return True + if (not self_stable) and other_stable: + return False + # Case 4, if two configs have a non-stable version in proto_path, + # the one with higher version is smaller. + # Note that using string comparison may lead unexpected result, + # e.g., v1beta10 is smaller than v1beta2. + # In reality, however, there's unlikely that a library has >10 beta + # versions. + if (not self_stable) and (not other_stable): + return self_version > other_version + # Two configs both have a stable version in proto_path. + # Case 5, if two configs have different depth in proto_path, the one + # with lower depth is smaller. + if len(self_dirs) != len(other_dirs): + return len(self_dirs) < len(other_dirs) + # Case 6, the config with higher stable version is smaller. + self_num = int(self_version[1:]) + other_num = int(other_version[1:]) + return self_num > other_num diff --git a/hermetic_build/common/model/generation_config.py b/hermetic_build/common/model/generation_config.py new file mode 100644 index 0000000000..bc3ff6d440 --- /dev/null +++ b/hermetic_build/common/model/generation_config.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import yaml +from typing import Optional +from library_generation.model.library_config import LibraryConfig +from library_generation.model.gapic_config import GapicConfig + +REPO_LEVEL_PARAMETER = "Repo level parameter" +LIBRARY_LEVEL_PARAMETER = "Library level parameter" +GAPIC_LEVEL_PARAMETER = "GAPIC level parameter" +COMMON_PROTOS_LIBRARY_NAME = "common-protos" +GAPIC_GENERATOR_VERSION = "gapic_generator_version" +LIBRARIES_BOM_VERSION = "libraries_bom_version" +GENERATOR_VERSION_ENV_KEY = "GENERATOR_VERSION" + + +class GenerationConfig: + """ + Class that represents the root of a generation_config.yaml + """ + + def __init__( + self, + googleapis_commitish: str, + libraries: list[LibraryConfig], + gapic_generator_version: Optional[str] = None, + libraries_bom_version: Optional[str] = None, + grpc_version: Optional[str] = None, + protoc_version: Optional[str] = None, + ): + self.googleapis_commitish = googleapis_commitish + self.libraries_bom_version = ( + libraries_bom_version if libraries_bom_version else "" + ) + self.gapic_generator_version = GenerationConfig.__set_generator_version( + gapic_generator_version + ) + self.libraries = libraries + self.grpc_version = grpc_version + self.protoc_version = protoc_version + # explicit set to None so that we can compute the + # value in getter. + self.__contains_common_protos = None + self.__validate() + + def get_proto_path_to_library_name(self) -> dict[str, str]: + """ + Get versioned proto_path to library_name mapping from configuration. + + :return: versioned proto_path to library_name mapping + """ + paths = {} + for library in self.libraries: + for gapic_config in library.gapic_configs: + paths[gapic_config.proto_path] = library.get_library_name() + return paths + + def is_monorepo(self) -> bool: + return len(self.libraries) > 1 + + def contains_common_protos(self) -> bool: + if self.__contains_common_protos is None: + self.__contains_common_protos = False + for library in self.libraries: + if library.get_library_name() == COMMON_PROTOS_LIBRARY_NAME: + self.__contains_common_protos = True + break + return self.__contains_common_protos + + @staticmethod + def __set_generator_version(gapic_generator_version: Optional[str]) -> str: + if gapic_generator_version is not None: + return gapic_generator_version + # if the generator version is not set through generation config, + # get it from environment variable. + gapic_generator_version = os.getenv(GENERATOR_VERSION_ENV_KEY) + if not gapic_generator_version: + raise ValueError( + f"Environment variable {GENERATOR_VERSION_ENV_KEY}" + f" is not set when the generator version is not" + f" specified in the generation config." + ) + return gapic_generator_version + + def __validate(self) -> None: + seen_library_names = dict() + for library in self.libraries: + library_name = library.get_library_name() + if library_name in seen_library_names: + raise ValueError( + f"Both {library.name_pretty} and " + f"{seen_library_names.get(library_name)} have the same " + f"library name: {library_name}, please update one of the " + f"library to have a different library name." + ) + seen_library_names[library_name] = library.name_pretty + + +def from_yaml(path_to_yaml: str) -> GenerationConfig: + """ + Parses a yaml located in path_to_yaml. + :param path_to_yaml: the path to the configuration file + :return the parsed configuration represented by the "model" classes + """ + with open(path_to_yaml, "r") as file_stream: + config = yaml.safe_load(file_stream) + + libraries = __required(config, "libraries", REPO_LEVEL_PARAMETER) + if not libraries: + raise ValueError(f"Library is None in {path_to_yaml}.") + + parsed_libraries = list() + for library in libraries: + gapics = __required(library, "GAPICs") + + parsed_gapics = list() + if not gapics: + raise ValueError(f"GAPICs is None in {library}.") + for gapic in gapics: + proto_path = __required(gapic, "proto_path", GAPIC_LEVEL_PARAMETER) + new_gapic = GapicConfig(proto_path) + parsed_gapics.append(new_gapic) + + new_library = LibraryConfig( + api_shortname=__required(library, "api_shortname"), + api_description=__required(library, "api_description"), + name_pretty=__required(library, "name_pretty"), + product_documentation=__required(library, "product_documentation"), + gapic_configs=parsed_gapics, + library_type=__optional(library, "library_type", "GAPIC_AUTO"), + release_level=__optional(library, "release_level", "preview"), + api_id=__optional(library, "api_id", None), + api_reference=__optional(library, "api_reference", None), + codeowner_team=__optional(library, "codeowner_team", None), + excluded_poms=__optional(library, "excluded_poms", None), + excluded_dependencies=__optional(library, "excluded_dependencies", None), + client_documentation=__optional(library, "client_documentation", None), + distribution_name=__optional(library, "distribution_name", None), + googleapis_commitish=__optional(library, "googleapis_commitish", None), + group_id=__optional(library, "group_id", "com.google.cloud"), + issue_tracker=__optional(library, "issue_tracker", None), + library_name=__optional(library, "library_name", None), + rest_documentation=__optional(library, "rest_documentation", None), + rpc_documentation=__optional(library, "rpc_documentation", None), + cloud_api=__optional(library, "cloud_api", True), + requires_billing=__optional(library, "requires_billing", True), + extra_versioned_modules=__optional( + library, "extra_versioned_modules", None + ), + recommended_package=__optional(library, "recommended_package", None), + min_java_version=__optional(library, "min_java_version", None), + transport=__optional(library, "transport", None), + ) + parsed_libraries.append(new_library) + + parsed_config = GenerationConfig( + googleapis_commitish=__required( + config, "googleapis_commitish", REPO_LEVEL_PARAMETER + ), + gapic_generator_version=__optional(config, GAPIC_GENERATOR_VERSION, None), + grpc_version=__optional(config, "grpc_version", None), + protoc_version=__optional(config, "protoc_version", None), + libraries_bom_version=__optional(config, LIBRARIES_BOM_VERSION, None), + libraries=parsed_libraries, + ) + + return parsed_config + + +def __required(config: dict, key: str, level: str = LIBRARY_LEVEL_PARAMETER): + if key not in config: + message = ( + f"{level}, {key}, is not found in {config} in yaml." + if level != REPO_LEVEL_PARAMETER + else f"{level}, {key}, is not found in yaml." + ) + raise ValueError(message) + return config[key] + + +def __optional(config: dict, key: str, default: any): + if key not in config: + return default + return config[key] diff --git a/hermetic_build/common/model/library_config.py b/hermetic_build/common/model/library_config.py new file mode 100644 index 0000000000..f7992f47a2 --- /dev/null +++ b/hermetic_build/common/model/library_config.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from hashlib import sha256 +from typing import Optional +from library_generation.model.gapic_config import GapicConfig +from library_generation.model.gapic_inputs import GapicInputs + + +MAVEN_COORDINATE_SEPARATOR = ":" + + +class LibraryConfig: + """ + Class that represents a library in a generation_config.yaml file + """ + + def __init__( + self, + api_shortname: str, + api_description: str, + name_pretty: str, + product_documentation: str, + gapic_configs: list[GapicConfig], + library_type: Optional[str] = None, + release_level: Optional[str] = None, + api_id: Optional[str] = None, + api_reference: Optional[str] = None, + codeowner_team: Optional[str] = None, + client_documentation: Optional[str] = None, + distribution_name: Optional[str] = None, + excluded_dependencies: Optional[str] = None, + excluded_poms: Optional[str] = None, + googleapis_commitish: Optional[str] = None, + group_id: Optional[str] = "com.google.cloud", + issue_tracker: Optional[str] = None, + library_name: Optional[str] = None, + rest_documentation: Optional[str] = None, + rpc_documentation: Optional[str] = None, + cloud_api: Optional[bool] = True, + requires_billing: Optional[bool] = True, + extra_versioned_modules: Optional[str] = None, + recommended_package: Optional[str] = None, + min_java_version: Optional[int] = None, + transport: Optional[str] = None, + ): + self.api_shortname = api_shortname + self.api_description = api_description + self.name_pretty = name_pretty + self.product_documentation = product_documentation + self.gapic_configs = gapic_configs + self.library_type = library_type if library_type else "GAPIC_AUTO" + self.release_level = release_level if release_level else "preview" + self.api_id = api_id + self.api_reference = api_reference + self.codeowner_team = codeowner_team + self.excluded_dependencies = excluded_dependencies + self.excluded_poms = excluded_poms + self.client_documentation = client_documentation + self.googleapis_commitish = googleapis_commitish + self.group_id = group_id + self.issue_tracker = issue_tracker + self.library_name = library_name + self.rest_documentation = rest_documentation + self.rpc_documentation = rpc_documentation + self.cloud_api = cloud_api + self.requires_billing = requires_billing + self.extra_versioned_modules = extra_versioned_modules + self.recommended_package = recommended_package + self.min_java_version = min_java_version + self.distribution_name = self.__get_distribution_name(distribution_name) + self.transport = self.__validate_transport(transport) + + def get_library_name(self) -> str: + """ + Return the library name of a given LibraryConfig object + :return: the library name + """ + return self.library_name if self.library_name else self.api_shortname + + def get_sorted_gapic_configs(self) -> list[GapicConfig]: + return sorted(self.gapic_configs) + + def get_maven_coordinate(self) -> str: + """ + Returns the Maven coordinate (group_id:artifact_id) of the library + """ + return self.distribution_name + + def get_artifact_id(self) -> str: + """ + Returns the artifact ID of the library + """ + return self.get_maven_coordinate().split(MAVEN_COORDINATE_SEPARATOR)[-1] + + def get_transport(self, gapic_inputs: GapicInputs) -> str: + """ + Returns the transport of the library. If directly set in library config, return it. + Otherwise, return the transport inferred from gapic_inputs. This value is only + used for postprocessing - the generation still infers the transport from BUILD + files. + """ + return self.transport if self.transport is not None else gapic_inputs.transport + + def __get_distribution_name(self, distribution_name: Optional[str]) -> str: + LibraryConfig.__check_distribution_name(distribution_name) + if distribution_name: + return distribution_name + cloud_prefix = "cloud-" if self.cloud_api else "" + library_name = self.get_library_name() + return f"{self.group_id}:google-{cloud_prefix}{library_name}" + + def __validate_transport(self, transport: str): + if transport not in [None, "grpc", "rest", "both"]: + raise ValueError( + "allowed values for library.transport: grpc, rest, both and None" + ) + return transport + + @staticmethod + def __check_distribution_name(distribution_name: str) -> None: + if not distribution_name: + return + sections = distribution_name.split(MAVEN_COORDINATE_SEPARATOR) + if len(sections) != 2: + raise ValueError(f"{distribution_name} is not a valid distribution name.") + + def __eq__(self, other): + return ( + self.api_shortname == other.api_shortname + and self.api_description == other.api_description + and self.name_pretty == other.name_pretty + and self.product_documentation == other.product_documentation + and self.gapic_configs == other.gapic_configs + and self.library_type == other.library_type + and self.release_level == other.release_level + and self.api_id == other.api_id + and self.api_reference == other.api_reference + and self.codeowner_team == other.codeowner_team + and self.excluded_dependencies == other.excluded_dependencies + and self.excluded_poms == other.excluded_poms + and self.client_documentation == other.client_documentation + and self.distribution_name == other.distribution_name + and self.googleapis_commitish == other.googleapis_commitish + and self.group_id == other.group_id + and self.issue_tracker == other.issue_tracker + and self.library_name == other.library_name + and self.rest_documentation == other.rest_documentation + and self.rpc_documentation == other.rpc_documentation + and self.cloud_api == other.cloud_api + and self.requires_billing == other.requires_billing + and self.extra_versioned_modules == other.extra_versioned_modules + and self.recommended_package == other.recommended_package + and self.min_java_version == other.min_java_version + and self.transport == other.transport + ) + + def __hash__(self): + m = sha256() + m.update( + str( + [ + self.api_shortname, + self.api_description, + self.name_pretty, + self.product_documentation, + self.library_type, + self.release_level, + self.api_id, + self.api_reference, + self.codeowner_team, + self.excluded_dependencies, + self.excluded_poms, + self.client_documentation, + self.distribution_name, + self.googleapis_commitish, + self.group_id, + self.issue_tracker, + self.library_name, + self.rest_documentation, + self.rpc_documentation, + self.cloud_api, + self.requires_billing, + self.extra_versioned_modules, + self.recommended_package, + self.min_java_version, + self.transport, + ] + + [config.proto_path for config in self.gapic_configs] + ).encode("utf-8") + ) + return int(m.hexdigest(), 16) diff --git a/hermetic_build/common/setup.py b/hermetic_build/common/setup.py new file mode 100755 index 0000000000..1adf08151a --- /dev/null +++ b/hermetic_build/common/setup.py @@ -0,0 +1,17 @@ +""" +Package information of library_generation python scripts +""" + +from setuptools import setup + +setup( + name="common", + version="0.1", + python_requires=">=3.12", + package_dir={ + "common": ".", + }, + install_requires=[ + "PyYAML==6.0.2", + ], +) diff --git a/hermetic_build/common/tests/model/gapic_config_unit_tests.py b/hermetic_build/common/tests/model/gapic_config_unit_tests.py new file mode 100644 index 0000000000..64d8556648 --- /dev/null +++ b/hermetic_build/common/tests/model/gapic_config_unit_tests.py @@ -0,0 +1,122 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.model.gapic_config import GapicConfig + + +class GapicConfigTest(unittest.TestCase): + def test_get_version_returns_none(self): + self.assertIsNone(GapicConfig(proto_path="example/dir1/dir2").get_version()) + + def test_get_version_returns_non_stable_version(self): + self.assertEqual( + "v2p2beta1", + GapicConfig(proto_path="example/dir1/dir2/v2p2beta1").get_version(), + ) + self.assertEqual( + "v2beta1", + GapicConfig(proto_path="example/dir1/dir2/v2beta1").get_version(), + ) + + def test_get_version_returns_stable_version(self): + self.assertEqual( + "v20", + GapicConfig(proto_path="example/dir1/dir2/v20").get_version(), + ) + + def test_is_stable_with_no_version_returns_false(self): + self.assertFalse( + GapicConfig(proto_path="example/dir1/dir2/non_version").is_stable(), + ) + + def test_is_stable_with_non_stable_version_returns_false(self): + self.assertFalse( + GapicConfig(proto_path="example/dir1/dir2/v20alpha").is_stable(), + ) + self.assertFalse( + GapicConfig(proto_path="example/dir1/dir2/v20beta2").is_stable(), + ) + + def test_is_stable_with_stable_version_returns_true(self): + self.assertTrue( + GapicConfig(proto_path="example/dir1/dir2/v30").is_stable(), + ) + + def test_compare_configs_without_a_version(self): + config_len_3 = GapicConfig(proto_path="example/dir1/dir2") + config_len_4 = GapicConfig(proto_path="example/dir1/dir2/dir3") + self.assertLess( + config_len_3, + config_len_4, + "config_len_3 should be smaller since it has a lower depth.", + ) + + def test_compare_configs_only_one_has_a_stable_version(self): + versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v1") + non_versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3") + self.assertLess( + versioned_config, + non_versioned_config, + "versioned_config should be smaller since it has a version.", + ) + + def test_compare_configs_only_one_has_a_non_stable_version(self): + non_stable_versioned_config = GapicConfig( + proto_path="example/dir1/dir2/dir3/dir4/v1beta" + ) + non_versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3") + self.assertLess( + non_stable_versioned_config, + non_versioned_config, + "non_stable_versioned_config should be smaller since it has a version.", + ) + + def test_compare_configs_one_has_non_stable_and_one_has_stable_version(self): + stable_versioned_config = GapicConfig( + proto_path="example/dir1/dir2/dir3/dir4/v1" + ) + non_stable_versioned_config = GapicConfig(proto_path="example/dir1/dir2/v2beta") + self.assertLess( + stable_versioned_config, + non_stable_versioned_config, + "stable_versioned_config should be smaller since it has a stable version.", + ) + + def test_compare_configs_two_have_non_stable_version(self): + v3p2beta = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v3p2beta") + v2p4beta = GapicConfig(proto_path="example/dir1/dir2/v2p4beta") + self.assertLess( + v3p2beta, + v2p4beta, + "v3p2beta should be smaller since it has a higher version.", + ) + + def test_compare_configs_two_have_stable_version_different_depth(self): + v3 = GapicConfig(proto_path="example/dir1/dir2/v3") + v4 = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v4") + self.assertLess( + v3, + v4, + "v3 should be smaller since it has lower depth", + ) + + def test_compare_configs_two_have_stable_version_same_depth(self): + v4 = GapicConfig(proto_path="example/dir1/dir2/v4") + v3 = GapicConfig(proto_path="example/dir1/dir2/v3") + self.assertLess( + v4, + v3, + "v4 should be smaller since it has a higher version", + ) diff --git a/hermetic_build/common/tests/model/generation_config_unit_tests.py b/hermetic_build/common/tests/model/generation_config_unit_tests.py new file mode 100644 index 0000000000..f6a2f2a2d8 --- /dev/null +++ b/hermetic_build/common/tests/model/generation_config_unit_tests.py @@ -0,0 +1,259 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +from pathlib import Path +from library_generation.model.generation_config import from_yaml, GenerationConfig +from library_generation.model.library_config import LibraryConfig + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") +test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() + +library_1 = LibraryConfig( + api_shortname="a_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[], +) +library_2 = LibraryConfig( + api_shortname="another_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[], +) +common_protos_library = LibraryConfig( + api_shortname="common-protos", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[], +) + + +class GenerationConfigTest(unittest.TestCase): + def test_generation_config_default_value(self): + config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + libraries=[], + ) + self.assertEqual("", config.libraries_bom_version) + + def test_generation_config_with_generator_version_env_raise_exception(self): + self.assertRaisesRegex( + ValueError, + "Environment variable GENERATOR_VERSION is not set", + GenerationConfig, + googleapis_commitish="", + libraries=[], + ) + + def test_generation_config_set_generator_version_from_env(self): + os.environ["GENERATOR_VERSION"] = "1.2.3" + config = GenerationConfig( + googleapis_commitish="", + libraries=[], + ) + self.assertEqual("1.2.3", config.gapic_generator_version) + os.environ.pop("GENERATOR_VERSION") + + def test_from_yaml_succeeds(self): + config = from_yaml(f"{test_config_dir}/generation_config.yaml") + self.assertEqual("2.34.0", config.gapic_generator_version) + self.assertEqual(25.2, config.protoc_version) + self.assertEqual( + "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish + ) + self.assertEqual("26.37.0", config.libraries_bom_version) + library = config.libraries[0] + self.assertEqual("cloudasset", library.api_shortname) + self.assertEqual("Cloud Asset Inventory", library.name_pretty) + self.assertEqual( + "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + library.product_documentation, + ) + self.assertEqual( + "provides inventory services based on a time series database.", + library.api_description, + ) + self.assertEqual("asset", library.library_name) + self.assertEqual("@googleapis/analytics-dpe", library.codeowner_team) + self.assertEqual( + "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1", + library.excluded_poms, + ) + self.assertEqual("google-iam-policy", library.excluded_dependencies) + gapics = library.gapic_configs + self.assertEqual(5, len(gapics)) + self.assertEqual("google/cloud/asset/v1", gapics[0].proto_path) + self.assertEqual("google/cloud/asset/v1p1beta1", gapics[1].proto_path) + self.assertEqual("google/cloud/asset/v1p2beta1", gapics[2].proto_path) + self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path) + self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path) + + def test_get_proto_path_to_library_name_success(self): + paths = from_yaml( + f"{test_config_dir}/generation_config.yaml" + ).get_proto_path_to_library_name() + self.assertEqual( + { + "google/cloud/asset/v1": "asset", + "google/cloud/asset/v1p1beta1": "asset", + "google/cloud/asset/v1p2beta1": "asset", + "google/cloud/asset/v1p5beta1": "asset", + "google/cloud/asset/v1p7beta1": "asset", + }, + paths, + ) + + def test_is_monorepo_with_one_library_returns_false(self): + config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + libraries=[library_1], + ) + self.assertFalse(config.is_monorepo()) + + def test_is_monorepo_with_two_libraries_returns_true(self): + config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + libraries=[library_1, library_2], + ) + self.assertTrue(config.is_monorepo()) + + def test_contains_common_protos_with_common_protos_returns_true(self): + config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + libraries=[library_1, library_2, common_protos_library], + ) + self.assertTrue(config.contains_common_protos()) + + def test_contains_common_protos_without_common_protos_returns_false(self): + config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + libraries=[library_1, library_2], + ) + self.assertFalse(config.contains_common_protos()) + + def test_validate_with_duplicate_library_name_raise_exception(self): + self.assertRaisesRegex( + ValueError, + "the same library name", + GenerationConfig, + gapic_generator_version="", + googleapis_commitish="", + libraries=[ + LibraryConfig( + api_shortname="secretmanager", + name_pretty="Secret API", + product_documentation="", + api_description="", + gapic_configs=list(), + ), + LibraryConfig( + api_shortname="another-secret", + name_pretty="Another Secret API", + product_documentation="", + api_description="", + gapic_configs=list(), + library_name="secretmanager", + ), + ], + ) + + def test_from_yaml_without_googleapis_commitish_raise_exception(self): + self.assertRaisesRegex( + ValueError, + "Repo level parameter, googleapis_commitish", + from_yaml, + f"{test_config_dir}/config_without_googleapis.yaml", + ) + + def test_from_yaml_without_libraries_raise_exception(self): + self.assertRaisesRegex( + ValueError, + "Repo level parameter, libraries", + from_yaml, + f"{test_config_dir}/config_without_libraries.yaml", + ) + + def test_from_yaml_without_api_shortname_raise_exception(self): + self.assertRaisesRegex( + ValueError, + "Library level parameter, api_shortname", + from_yaml, + f"{test_config_dir}/config_without_api_shortname.yaml", + ) + + def test_from_yaml_without_api_description_raise_exception(self): + self.assertRaisesRegex( + ValueError, + r"Library level parameter, api_description.*'api_shortname': 'apigeeconnect'.*", + from_yaml, + f"{test_config_dir}/config_without_api_description.yaml", + ) + + def test_from_yaml_without_name_pretty_raise_exception(self): + self.assertRaisesRegex( + ValueError, + r"Library level parameter, name_pretty.*'api_shortname': 'apigeeconnect'.*", + from_yaml, + f"{test_config_dir}/config_without_name_pretty.yaml", + ) + + def test_from_yaml_without_product_documentation_raise_exception(self): + self.assertRaisesRegex( + ValueError, + r"Library level parameter, product_documentation.*'api_shortname': 'apigeeconnect'.*", + from_yaml, + f"{test_config_dir}/config_without_product_docs.yaml", + ) + + def test_from_yaml_without_gapics_raise_exception(self): + self.assertRaisesRegex( + ValueError, + "Library level parameter, GAPICs.*'api_shortname': 'apigeeconnect'.*", + from_yaml, + f"{test_config_dir}/config_without_gapics_key.yaml", + ) + + def test_from_yaml_without_proto_path_raise_exception(self): + self.assertRaisesRegex( + ValueError, + "GAPIC level parameter, proto_path", + from_yaml, + f"{test_config_dir}/config_without_proto_path.yaml", + ) + + def test_from_yaml_with_zero_library_raise_exception(self): + self.assertRaisesRegex( + ValueError, + "Library is None", + from_yaml, + f"{test_config_dir}/config_without_library_value.yaml", + ) + + def test_from_yaml_with_zero_proto_path_raise_exception(self): + self.assertRaisesRegex( + ValueError, + r"GAPICs is None in.*'api_shortname': 'apigeeconnect'.*", + from_yaml, + f"{test_config_dir}/config_without_gapics_value.yaml", + ) diff --git a/hermetic_build/common/tests/model/library_config_unit_tests.py b/hermetic_build/common/tests/model/library_config_unit_tests.py new file mode 100644 index 0000000000..5d54737ced --- /dev/null +++ b/hermetic_build/common/tests/model/library_config_unit_tests.py @@ -0,0 +1,122 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.model.gapic_config import GapicConfig +from library_generation.model.library_config import LibraryConfig + + +class LibraryConfigTest(unittest.TestCase): + def test_get_library_returns_library_name(self): + library = LibraryConfig( + api_shortname="secret", + name_pretty="", + product_documentation="", + api_description="", + gapic_configs=list(), + library_name="secretmanager", + ) + self.assertEqual("secretmanager", library.get_library_name()) + + def test_get_library_returns_api_shortname(self): + library = LibraryConfig( + api_shortname="secret", + name_pretty="", + product_documentation="", + api_description="", + gapic_configs=list(), + ) + self.assertEqual("secret", library.get_library_name()) + + def test_get_sorted_gapic_configs_returns_correct_order(self): + v1beta1 = GapicConfig(proto_path="google/spanner/v1beta1") + v1 = GapicConfig(proto_path="google/spanner/v1") + v1alpha1 = GapicConfig(proto_path="google/spanner/v1alpha") + v2 = GapicConfig(proto_path="google/spanner/v2") + admin_v2 = GapicConfig(proto_path="google/spanner/admin/v2") + non_versioned = GapicConfig(proto_path="google/spanner/type") + library = LibraryConfig( + api_shortname="secret", + name_pretty="", + product_documentation="", + api_description="", + gapic_configs=[v1alpha1, v1, v2, admin_v2, non_versioned, v1beta1], + ) + self.assertEqual( + [ + v2, + v1, + admin_v2, + v1beta1, + v1alpha1, + non_versioned, + ], + library.get_sorted_gapic_configs(), + ) + + def test_init_invalid_distribution_name_raise_value_error(self): + self.assertRaisesRegex( + ValueError, + "fake-distribution-name is not a valid distribution name.", + LibraryConfig, + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + distribution_name="fake-distribution-name", + ) + + def test_get_distribution_name_cloud_api(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + ) + self.assertEqual( + "com.google.cloud:google-cloud-baremetalsolution", + library.get_maven_coordinate(), + ) + self.assertEqual("google-cloud-baremetalsolution", library.get_artifact_id()) + + def test_get_distribution_name_non_cloud_api(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + cloud_api=False, + group_id="com.example", + ) + self.assertEqual( + "com.example:google-baremetalsolution", library.get_maven_coordinate() + ) + self.assertEqual("google-baremetalsolution", library.get_artifact_id()) + + def test_get_distribution_name_with_distribution_name(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + distribution_name="com.example:baremetalsolution", + ) + self.assertEqual( + "com.example:baremetalsolution", library.get_maven_coordinate() + ) + self.assertEqual("baremetalsolution", library.get_artifact_id()) diff --git a/hermetic_build/common/tests/resources/test-config/config_without_api_description.yaml b/hermetic_build/common/tests/resources/test-config/config_without_api_description.yaml new file mode 100644 index 0000000000..79ff135067 --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_api_description.yaml @@ -0,0 +1,5 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/common/tests/resources/test-config/config_without_api_shortname.yaml b/hermetic_build/common/tests/resources/test-config/config_without_api_shortname.yaml new file mode 100644 index 0000000000..ec8206be61 --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_api_shortname.yaml @@ -0,0 +1,4 @@ +gapic_generator_version: 2.34.0 +libraries: + - GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/common/tests/resources/test-config/config_without_gapics_key.yaml b/hermetic_build/common/tests/resources/test-config/config_without_gapics_key.yaml new file mode 100644 index 0000000000..739a4d9239 --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_gapics_key.yaml @@ -0,0 +1,3 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect diff --git a/hermetic_build/common/tests/resources/test-config/config_without_gapics_value.yaml b/hermetic_build/common/tests/resources/test-config/config_without_gapics_value.yaml new file mode 100644 index 0000000000..ec49e4a669 --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_gapics_value.yaml @@ -0,0 +1,4 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect + GAPICs: diff --git a/hermetic_build/common/tests/resources/test-config/config_without_googleapis.yaml b/hermetic_build/common/tests/resources/test-config/config_without_googleapis.yaml new file mode 100644 index 0000000000..e5a00ca4ee --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_googleapis.yaml @@ -0,0 +1,8 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + api_description: "allows the Apigee hybrid management" + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/common/tests/resources/test-config/config_without_libraries.yaml b/hermetic_build/common/tests/resources/test-config/config_without_libraries.yaml new file mode 100644 index 0000000000..dbbe2ea318 --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_libraries.yaml @@ -0,0 +1 @@ +gapic_generator_version: 2.34.0 diff --git a/hermetic_build/common/tests/resources/test-config/config_without_library_value.yaml b/hermetic_build/common/tests/resources/test-config/config_without_library_value.yaml new file mode 100644 index 0000000000..174a293000 --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_library_value.yaml @@ -0,0 +1,2 @@ +gapic_generator_version: 2.34.0 +libraries: diff --git a/hermetic_build/common/tests/resources/test-config/config_without_name_pretty.yaml b/hermetic_build/common/tests/resources/test-config/config_without_name_pretty.yaml new file mode 100644 index 0000000000..f8612ad9ca --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_name_pretty.yaml @@ -0,0 +1,6 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect + api_description: "allows the Apigee hybrid management" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/common/tests/resources/test-config/config_without_product_docs.yaml b/hermetic_build/common/tests/resources/test-config/config_without_product_docs.yaml new file mode 100644 index 0000000000..e3921d2c0d --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_product_docs.yaml @@ -0,0 +1,7 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + api_description: "allows the Apigee hybrid management" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/common/tests/resources/test-config/config_without_proto_path.yaml b/hermetic_build/common/tests/resources/test-config/config_without_proto_path.yaml new file mode 100644 index 0000000000..e37b0cef63 --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_proto_path.yaml @@ -0,0 +1,4 @@ +gapic_generator_version: 2.34.0 +libraries: + - GAPICs: + - random_key: diff --git a/hermetic_build/common/tests/resources/test-config/config_without_temp_excludes.yaml b/hermetic_build/common/tests/resources/test-config/config_without_temp_excludes.yaml new file mode 100644 index 0000000000..0d1bb7deea --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/config_without_temp_excludes.yaml @@ -0,0 +1,10 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + api_description: "allows the Apigee hybrid management" + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/common/tests/resources/test-config/generation_config.yaml b/hermetic_build/common/tests/resources/test-config/generation_config.yaml new file mode 100644 index 0000000000..168c8fd9a5 --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/generation_config.yaml @@ -0,0 +1,24 @@ +gapic_generator_version: 2.34.0 +protoc_version: 25.2 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: "stable" + issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" + api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + codeowner_team: "@googleapis/analytics-dpe" + excluded_poms: proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1 + excluded_dependencies: google-iam-policy + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 diff --git a/hermetic_build/common/tests/resources/test-config/generation_config_library_modified.yaml b/hermetic_build/common/tests/resources/test-config/generation_config_library_modified.yaml new file mode 100644 index 0000000000..f9ae96693b --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/generation_config_library_modified.yaml @@ -0,0 +1,14 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 diff --git a/hermetic_build/common/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml b/hermetic_build/common/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml new file mode 100644 index 0000000000..c5613f4308 --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml @@ -0,0 +1,51 @@ +gapic_generator_version: 2.34.0 +protoc_version: 25.2 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 +synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 +template_excludes: + - ".github/*" + - ".kokoro/*" + - "samples/*" + - "CODE_OF_CONDUCT.md" + - "CONTRIBUTING.md" + - "LICENSE" + - "SECURITY.md" + - "java.header" + - "license-checks.xml" + - "renovate.json" + - ".gitignore" +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: "stable" + issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" + api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + codeowner_team: "@googleapis/analytics-dpe" + excluded_poms: proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1 + excluded_dependencies: google-iam-policy + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + + - api_shortname: another-cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: "stable" + issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" + api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + GAPICs: + - proto_path: google/cloud/asset/v1 diff --git a/hermetic_build/common/tests/resources/test-config/monorepo_baseline.yaml b/hermetic_build/common/tests/resources/test-config/monorepo_baseline.yaml new file mode 100644 index 0000000000..c2c4fd4a3b --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/monorepo_baseline.yaml @@ -0,0 +1,30 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: preview + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + - api_shortname: cloudbuild + name_pretty: Cloud Build + product_documentation: https://cloud.google.com/cloud-build/ + api_description: lets you build software quickly across all languages. Get complete + control over defining custom workflows for building, testing, and deploying across + multiple environments such as VMs, serverless, Kubernetes, or Firebase. + release_level: stable + distribution_name: com.google.cloud:google-cloud-build + issue_tracker: https://issuetracker.google.com/savedsearches/5226584 + GAPICs: + - proto_path: google/devtools/cloudbuild/v1 + - proto_path: google/devtools/cloudbuild/v2 diff --git a/hermetic_build/common/tests/resources/test-config/monorepo_current.yaml b/hermetic_build/common/tests/resources/test-config/monorepo_current.yaml new file mode 100644 index 0000000000..3ee2c8be2c --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/monorepo_current.yaml @@ -0,0 +1,30 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: stable + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + - api_shortname: cloudbuild + name_pretty: Cloud Build + product_documentation: https://cloud.google.com/cloud-build/ + api_description: lets you build software quickly across all languages. Get complete + control over defining custom workflows for building, testing, and deploying across + multiple environments such as VMs, serverless, Kubernetes, or Firebase. + release_level: stable + distribution_name: com.google.cloud:google-cloud-build + issue_tracker: https://issuetracker.google.com/savedsearches/5226584 + GAPICs: + - proto_path: google/devtools/cloudbuild/v1 + - proto_path: google/devtools/cloudbuild/v2 diff --git a/hermetic_build/common/tests/resources/test-config/monorepo_with_common_protos.yaml b/hermetic_build/common/tests/resources/test-config/monorepo_with_common_protos.yaml new file mode 100644 index 0000000000..6d4c94444a --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/monorepo_with_common_protos.yaml @@ -0,0 +1,43 @@ +googleapis_commitish: 6a474b31c53cc1797710206824a17b364a835d2d +gapic_generator_version: 2.34.0 +# the libraries are ordered with respect to library name, which is +# java-{library.library_name} or java-{library.api-shortname} when +# library.library_name is not defined. +libraries: +- api_shortname: common-protos + name_pretty: Common Protos + product_documentation: https://github.com/googleapis/api-common-protos + api_description: Protobuf classes for Google's common protos. + release_level: stable + client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history + distribution_name: com.google.api.grpc:proto-google-common-protos + excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" + excluded_poms: "proto-google-common-protos-bom,proto-google-common-protos" + library_type: OTHER + GAPICs: + - proto_path: google/api + - proto_path: google/apps/card/v1 + - proto_path: google/cloud + - proto_path: google/cloud/audit + - proto_path: google/cloud/location + - proto_path: google/geo/type + - proto_path: google/logging/type + - proto_path: google/longrunning + - proto_path: google/rpc + - proto_path: google/rpc/context + - proto_path: google/shopping/type + - proto_path: google/type +- api_shortname: iam + name_pretty: IAM + product_documentation: https://cloud.google.com/iam + api_description: Manages access control for Google Cloud Platform resources + release_level: stable + client_documentation: https://cloud.google.com/java/docs/reference/proto-google-iam-v1/latest/overview + distribution_name: com.google.api.grpc:proto-google-iam-v1 + excluded_dependencies: "grpc-google-iam-v1" + excluded_poms: "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1" + library_type: OTHER + GAPICs: + - proto_path: google/iam/v1 + - proto_path: google/iam/v2 + - proto_path: google/iam/v2beta \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/test-config/monorepo_without_common_protos.yaml b/hermetic_build/common/tests/resources/test-config/monorepo_without_common_protos.yaml new file mode 100644 index 0000000000..ca21ccfb01 --- /dev/null +++ b/hermetic_build/common/tests/resources/test-config/monorepo_without_common_protos.yaml @@ -0,0 +1,33 @@ +gapic_generator_version: 2.34.0 +protoc_version: 25.2 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: "stable" + issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" + api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + - api_shortname: cloudbuild + name_pretty: Cloud Build + product_documentation: https://cloud.google.com/cloud-build/ + api_description: lets you build software quickly across all languages. Get complete + control over defining custom workflows for building, testing, and deploying across + multiple environments such as VMs, serverless, Kubernetes, or Firebase. + release_level: stable + distribution_name: com.google.cloud:google-cloud-build + issue_tracker: https://issuetracker.google.com/savedsearches/5226584 + GAPICs: + - proto_path: google/devtools/cloudbuild/v1 + - proto_path: google/devtools/cloudbuild/v2 From 82c164299c29dc694aaf280add25879c6a281a37 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 09:53:16 -0400 Subject: [PATCH 14/69] fix package --- hermetic_build/common/tests/model/__init__.py | 0 hermetic_build/common/tests/model/gapic_config_unit_tests.py | 2 +- .../common/tests/model/generation_config_unit_tests.py | 4 ++-- .../common/tests/model/library_config_unit_tests.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 hermetic_build/common/tests/model/__init__.py diff --git a/hermetic_build/common/tests/model/__init__.py b/hermetic_build/common/tests/model/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/common/tests/model/gapic_config_unit_tests.py b/hermetic_build/common/tests/model/gapic_config_unit_tests.py index 64d8556648..864b2556e4 100644 --- a/hermetic_build/common/tests/model/gapic_config_unit_tests.py +++ b/hermetic_build/common/tests/model/gapic_config_unit_tests.py @@ -13,7 +13,7 @@ # limitations under the License. import unittest -from library_generation.model.gapic_config import GapicConfig +from common.model.gapic_config import GapicConfig class GapicConfigTest(unittest.TestCase): diff --git a/hermetic_build/common/tests/model/generation_config_unit_tests.py b/hermetic_build/common/tests/model/generation_config_unit_tests.py index f6a2f2a2d8..94edd934a0 100644 --- a/hermetic_build/common/tests/model/generation_config_unit_tests.py +++ b/hermetic_build/common/tests/model/generation_config_unit_tests.py @@ -14,8 +14,8 @@ import os import unittest from pathlib import Path -from library_generation.model.generation_config import from_yaml, GenerationConfig -from library_generation.model.library_config import LibraryConfig +from common.model.generation_config import from_yaml, GenerationConfig +from common.model.library_config import LibraryConfig script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, "..", "resources") diff --git a/hermetic_build/common/tests/model/library_config_unit_tests.py b/hermetic_build/common/tests/model/library_config_unit_tests.py index 5d54737ced..4935979ffa 100644 --- a/hermetic_build/common/tests/model/library_config_unit_tests.py +++ b/hermetic_build/common/tests/model/library_config_unit_tests.py @@ -13,8 +13,8 @@ # limitations under the License. import unittest -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.library_config import LibraryConfig +from common.model.gapic_config import GapicConfig +from common.model.library_config import LibraryConfig class LibraryConfigTest(unittest.TestCase): From 54baaf90fae6d9d4d89d6da5a9da780dbbbfa9e1 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 12:21:37 -0400 Subject: [PATCH 15/69] add gapic_inputs.py to common --- hermetic_build/common/model/gapic_inputs.py | 196 +++++++++++++ .../common/model/generation_config.py | 4 +- hermetic_build/common/model/library_config.py | 4 +- hermetic_build/common/setup.py | 1 + .../tests/model/gapic_inputs_unit_tests.py | 132 +++++++++ .../misc/BUILD_comment_common_resources.bazel | 5 + .../misc/BUILD_comment_iam_policy.bazel | 5 + .../misc/BUILD_comment_locations.bazel | 5 + .../misc/BUILD_common_resources.bazel | 5 + .../resources/misc/BUILD_gapic_yaml.bazel | 3 + .../tests/resources/misc/BUILD_grpc.bazel | 5 + .../resources/misc/BUILD_grpc_rest.bazel | 5 + .../resources/misc/BUILD_iam_locations.bazel | 6 + .../resources/misc/BUILD_iam_policy.bazel | 5 + .../misc/BUILD_include_samples_empty.bazel | 5 + .../misc/BUILD_include_samples_false.bazel | 5 + .../misc/BUILD_include_samples_true.bazel | 5 + .../resources/misc/BUILD_locations.bazel | 5 + .../misc/BUILD_no_additional_protos.bazel | 4 + .../resources/misc/BUILD_no_gapic_yaml.bazel | 3 + .../misc/BUILD_no_service_config.bazel | 3 + .../misc/BUILD_no_service_yaml.bazel | 3 + .../resources/misc/BUILD_proto_only.bazel | 16 ++ .../tests/resources/misc/BUILD_rest.bazel | 5 + .../misc/BUILD_rest_numeric_enums_empty.bazel | 5 + .../misc/BUILD_rest_numeric_enums_false.bazel | 5 + .../misc/BUILD_rest_numeric_enums_true.bazel | 5 + .../resources/misc/BUILD_service_config.bazel | 3 + ...BUILD_service_config_relative_target.bazel | 3 + .../resources/misc/BUILD_service_yaml.bazel | 3 + .../BUILD_service_yaml_absolute_target.bazel | 3 + .../common/tests/resources/misc/TESTWORKSPACE | 133 +++++++++ .../common/tests/resources/misc/versions.txt | 9 + .../release_note_generation/setup.py | 17 ++ .../release_note_generation/tests/__init__.py | 0 .../tests/cli/__init__.py | 0 .../cli/generate_release_note_unit_tests.py | 71 +++++ .../commit_message_formatter_unit_tests.py | 199 +++++++++++++ .../generate_pr_description_unit_tests.py | 270 ++++++++++++++++++ 39 files changed, 1157 insertions(+), 4 deletions(-) create mode 100644 hermetic_build/common/model/gapic_inputs.py create mode 100644 hermetic_build/common/tests/model/gapic_inputs_unit_tests.py create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_comment_common_resources.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_comment_iam_policy.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_comment_locations.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_common_resources.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_gapic_yaml.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_grpc.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_grpc_rest.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_iam_locations.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_iam_policy.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_include_samples_empty.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_include_samples_false.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_include_samples_true.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_locations.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_no_additional_protos.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_no_gapic_yaml.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_no_service_config.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_no_service_yaml.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_proto_only.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_rest.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_service_config.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_service_config_relative_target.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_service_yaml.bazel create mode 100644 hermetic_build/common/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel create mode 100644 hermetic_build/common/tests/resources/misc/TESTWORKSPACE create mode 100644 hermetic_build/common/tests/resources/misc/versions.txt create mode 100755 hermetic_build/release_note_generation/setup.py create mode 100644 hermetic_build/release_note_generation/tests/__init__.py create mode 100644 hermetic_build/release_note_generation/tests/cli/__init__.py create mode 100644 hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py create mode 100644 hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py create mode 100644 hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py diff --git a/hermetic_build/common/model/gapic_inputs.py b/hermetic_build/common/model/gapic_inputs.py new file mode 100644 index 0000000000..5c07777965 --- /dev/null +++ b/hermetic_build/common/model/gapic_inputs.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import re + +proto_library_pattern = r""" +proto_library_with_info\( +(.*?) +\) +""" +gapic_pattern = r""" +java_gapic_library\( +(.*?) +\) +""" +assembly_pattern = r""" +java_gapic_assembly_gradle_pkg\( +(.*?) +\) +""" +# match a line which the first character is "#". +comment_pattern = r"^\s*\#+" +pattern_to_proto = { + r"//google/cloud:common_resources_proto": "google/cloud/common_resources.proto", + r"//google/cloud/location:location_proto": "google/cloud/location/locations.proto", + r"//google/iam/v1:iam_policy_proto": "google/iam/v1/iam_policy.proto", +} +transport_pattern = r"transport = \"(.*?)\"" +rest_pattern = r"rest_numeric_enums = True" +gapic_yaml_pattern = r"gapic_yaml = \"(.*?)\"" +service_config_pattern = r"grpc_service_config = \"(.*?)\"" +service_yaml_pattern = r"service_yaml = \"(.*?)\"" +include_samples_pattern = r"include_samples = True" + + +class GapicInputs: + """ + A data class containing inputs to invoke generate_library.sh to generate + a GAPIC library. + """ + + def __init__( + self, + proto_only="true", + additional_protos="google/cloud/common_resources.proto", + transport="grpc", + rest_numeric_enum="", + gapic_yaml="", + service_config="", + service_yaml="", + include_samples="true", + ): + self.proto_only = proto_only + self.additional_protos = additional_protos + self.transport = transport + self.rest_numeric_enum = rest_numeric_enum + self.gapic_yaml = gapic_yaml + self.service_config = service_config + self.service_yaml = service_yaml + self.include_samples = include_samples + + def __eq__(self, other): + if not isinstance(other, GapicInputs): + return False + return ( + self.proto_only == other.proto_only + and self.additional_protos == other.additional_protos + and self.transport == other.transport + and self.rest_numeric_enum == other.rest_numeric_enum + and self.gapic_yaml == other.gapic_yaml + and self.service_config == other.service_config + and self.service_yaml == other.service_yaml + and self.include_samples == other.include_samples + ) + + +def parse( + build_path: Path, versioned_path: str, build_file_name: str = "BUILD.bazel" +) -> GapicInputs: + """ + Utility function to parse inputs of generate_library.sh from BUILD.bazel. + :param build_path: the file path of BUILD.bazel + :param versioned_path: a versioned path in googleapis repository, e.g., + google/cloud/asset/v1. + :param build_file_name: the name of the build file. + :return: an GapicInputs object. + """ + with open(f"{build_path}/{build_file_name}") as build: + content = build.read() + return parse_build_str(build_str=content, versioned_path=versioned_path) + + +def parse_build_str(build_str: str, versioned_path: str) -> GapicInputs: + proto_library_target = re.compile( + proto_library_pattern, re.DOTALL | re.VERBOSE + ).findall(build_str) + additional_protos = "" + if len(proto_library_target) > 0: + additional_protos = __parse_additional_protos(proto_library_target[0]) + gapic_target = re.compile(gapic_pattern, re.DOTALL | re.VERBOSE).findall(build_str) + assembly_target = re.compile(assembly_pattern, re.DOTALL | re.VERBOSE).findall( + build_str + ) + include_samples = "false" + if len(assembly_target) > 0: + include_samples = __parse_include_samples(assembly_target[0]) + if len(gapic_target) == 0: + return GapicInputs( + additional_protos=additional_protos, include_samples=include_samples + ) + + transport = __parse_transport(gapic_target[0]) + rest_numeric_enum = __parse_rest_numeric_enums(gapic_target[0]) + gapic_yaml = __parse_gapic_yaml(gapic_target[0], versioned_path) + service_config = __parse_service_config(gapic_target[0], versioned_path) + service_yaml = __parse_service_yaml(gapic_target[0], versioned_path) + + return GapicInputs( + proto_only="false", + additional_protos=additional_protos, + transport=transport, + rest_numeric_enum=rest_numeric_enum, + gapic_yaml=gapic_yaml, + service_config=service_config, + service_yaml=service_yaml, + include_samples=include_samples, + ) + + +def __parse_additional_protos(proto_library_target: str) -> str: + res = [" "] + lines = proto_library_target.split("\n") + for line in lines: + if len(re.findall(comment_pattern, line)) != 0: + # skip a line which the first charactor is "#" since it's + # a comment. + continue + for pattern in pattern_to_proto: + if len(re.findall(pattern, line)) == 0: + continue + res.append(pattern_to_proto[pattern]) + return " ".join(res) + + +def __parse_transport(gapic_target: str) -> str: + transport = re.findall(transport_pattern, gapic_target) + return transport[0] if len(transport) != 0 else "grpc" + + +def __parse_rest_numeric_enums(gapic_target: str) -> str: + rest_numeric_enums = re.findall(rest_pattern, gapic_target) + return "true" if len(rest_numeric_enums) != 0 else "false" + + +def __parse_gapic_yaml(gapic_target: str, versioned_path: str) -> str: + gapic_yaml = re.findall(gapic_yaml_pattern, gapic_target) + return f"{versioned_path}/{gapic_yaml[0]}" if len(gapic_yaml) != 0 else "" + + +def __parse_service_config(gapic_target: str, versioned_path: str) -> str: + service_config = re.findall(service_config_pattern, gapic_target) + return ( + f"{versioned_path}/{service_config[0]}".replace(":", "") + if len(service_config) != 0 + else "" + ) + + +def __parse_service_yaml(gapic_target: str, versioned_path: str) -> str: + service_yaml = re.findall(service_yaml_pattern, gapic_target) + if len(service_yaml) == 0: + return "" + res = str(service_yaml[0]) + if res.startswith("//"): + # special case if the service config starts with "//", is a Bazel + # target with an absolute path. + return res.replace("//", "").replace(":", "/") + return f"{versioned_path}/{res}" + + +def __parse_include_samples(assembly_target: str) -> str: + include_samples = re.findall(include_samples_pattern, assembly_target) + return "true" if len(include_samples) != 0 else "false" diff --git a/hermetic_build/common/model/generation_config.py b/hermetic_build/common/model/generation_config.py index bc3ff6d440..562e20ce3f 100644 --- a/hermetic_build/common/model/generation_config.py +++ b/hermetic_build/common/model/generation_config.py @@ -16,8 +16,8 @@ import yaml from typing import Optional -from library_generation.model.library_config import LibraryConfig -from library_generation.model.gapic_config import GapicConfig +from common.model.library_config import LibraryConfig +from common.model.gapic_config import GapicConfig REPO_LEVEL_PARAMETER = "Repo level parameter" LIBRARY_LEVEL_PARAMETER = "Library level parameter" diff --git a/hermetic_build/common/model/library_config.py b/hermetic_build/common/model/library_config.py index f7992f47a2..b4156a6408 100644 --- a/hermetic_build/common/model/library_config.py +++ b/hermetic_build/common/model/library_config.py @@ -14,8 +14,8 @@ # limitations under the License. from hashlib import sha256 from typing import Optional -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.gapic_inputs import GapicInputs +from common.model.gapic_config import GapicConfig +from common.model.gapic_inputs import GapicInputs MAVEN_COORDINATE_SEPARATOR = ":" diff --git a/hermetic_build/common/setup.py b/hermetic_build/common/setup.py index 1adf08151a..5e2dc0f55a 100755 --- a/hermetic_build/common/setup.py +++ b/hermetic_build/common/setup.py @@ -12,6 +12,7 @@ "common": ".", }, install_requires=[ + "parameterized==0.9.0", "PyYAML==6.0.2", ], ) diff --git a/hermetic_build/common/tests/model/gapic_inputs_unit_tests.py b/hermetic_build/common/tests/model/gapic_inputs_unit_tests.py new file mode 100644 index 0000000000..41e9e7d233 --- /dev/null +++ b/hermetic_build/common/tests/model/gapic_inputs_unit_tests.py @@ -0,0 +1,132 @@ +import os +import unittest +from pathlib import Path + +from parameterized import parameterized +from common.model.gapic_inputs import parse + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") +build_file = Path(os.path.join(resources_dir, "misc")).resolve() + + +class UtilitiesTest(unittest.TestCase): + @parameterized.expand( + [ + ("BUILD_no_additional_protos.bazel", " "), + ("BUILD_common_resources.bazel", " google/cloud/common_resources.proto"), + ("BUILD_comment_common_resources.bazel", " "), + ("BUILD_locations.bazel", " google/cloud/location/locations.proto"), + ("BUILD_comment_locations.bazel", " "), + ("BUILD_iam_policy.bazel", " google/iam/v1/iam_policy.proto"), + ("BUILD_comment_iam_policy.bazel", " "), + ( + "BUILD_iam_locations.bazel", + " google/cloud/location/locations.proto google/iam/v1/iam_policy.proto", + ), + ] + ) + def test_gapic_inputs_parse_additional_protos(self, build_name, expected): + parsed = parse(build_file, "", build_name) + self.assertEqual( + expected, + parsed.additional_protos, + ) + + def test_gapic_inputs_parse_grpc_only_succeeds(self): + parsed = parse(build_file, "", "BUILD_grpc.bazel") + self.assertEqual("grpc", parsed.transport) + + def test_gapic_inputs_parse_grpc_rest_succeeds(self): + parsed = parse(build_file, "", "BUILD_grpc_rest.bazel") + self.assertEqual("grpc+rest", parsed.transport) + + def test_gapic_inputs_parse_rest_succeeds(self): + parsed = parse(build_file, "", "BUILD_rest.bazel") + self.assertEqual("rest", parsed.transport) + + def test_gapic_inputs_parse_empty_include_samples_succeeds(self): + parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel") + self.assertEqual("false", parsed.include_samples) + + def test_gapic_inputs_parse_include_samples_false_succeeds(self): + parsed = parse(build_file, "", "BUILD_include_samples_false.bazel") + self.assertEqual("false", parsed.include_samples) + + def test_gapic_inputs_parse_include_samples_true_succeeds(self): + parsed = parse(build_file, "", "BUILD_include_samples_true.bazel") + self.assertEqual("true", parsed.include_samples) + + def test_gapic_inputs_parse_empty_rest_numeric_enums_succeeds(self): + parsed = parse(build_file, "", "BUILD_rest_numeric_enums_empty.bazel") + self.assertEqual("false", parsed.rest_numeric_enum) + + def test_gapic_inputs_parse_rest_numeric_enums_false_succeeds(self): + parsed = parse(build_file, "", "BUILD_rest_numeric_enums_false.bazel") + self.assertEqual("false", parsed.rest_numeric_enum) + + def test_gapic_inputs_parse_rest_numeric_enums_true_succeeds(self): + parsed = parse(build_file, "", "BUILD_rest_numeric_enums_true.bazel") + self.assertEqual("true", parsed.rest_numeric_enum) + + def test_gapic_inputs_parse_no_gapic_library_returns_proto_only_true(self): + # include_samples_empty only has a gradle assembly rule + parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel") + self.assertEqual("true", parsed.proto_only) + + def test_gapic_inputs_parse_with_gapic_library_returns_proto_only_false(self): + # rest.bazel has a java_gapic_library rule + parsed = parse(build_file, "", "BUILD_rest.bazel") + self.assertEqual("false", parsed.proto_only) + + def test_gapic_inputs_parse_gapic_yaml_succeeds(self): + parsed = parse(build_file, "test/versioned/path", "BUILD_gapic_yaml.bazel") + self.assertEqual("test/versioned/path/test_gapic_yaml.yaml", parsed.gapic_yaml) + + def test_gapic_inputs_parse_no_gapic_yaml_returns_empty_string(self): + parsed = parse(build_file, "test/versioned/path", "BUILD_no_gapic_yaml.bazel") + self.assertEqual("", parsed.gapic_yaml) + + def test_gapic_inputs_parse_service_config_succeeds(self): + parsed = parse(build_file, "test/versioned/path", "BUILD_service_config.bazel") + self.assertEqual( + "test/versioned/path/test_service_config.json", parsed.service_config + ) + + def test_gapic_inputs_parse_service_yaml_relative_target(self): + parsed = parse( + build_file, + "google/cloud/compute/v1", + "BUILD_service_config_relative_target.bazel", + ) + self.assertEqual( + "google/cloud/compute/v1/compute_grpc_service_config.json", + parsed.service_config, + ) + + def test_gapic_inputs_parse_no_service_config_returns_empty_string(self): + parsed = parse( + build_file, "test/versioned/path", "BUILD_no_service_config.bazel" + ) + self.assertEqual("", parsed.service_config) + + def test_gapic_inputs_parse_service_yaml_succeeds(self): + parsed = parse(build_file, "test/versioned/path", "BUILD_service_yaml.bazel") + self.assertEqual( + "test/versioned/path/test_service_yaml.yaml", parsed.service_yaml + ) + + def test_gapic_inputs_parse_service_yaml_absolute_target(self): + parsed = parse(build_file, "", "BUILD_service_yaml_absolute_target.bazel") + self.assertEqual( + "google/cloud/videointelligence/videointelligence_v1p3beta1.yaml", + parsed.service_yaml, + ) + + def test_gapic_inputs_parse_no_service_yaml_returns_empty_string(self): + parsed = parse(build_file, "test/versioned/path", "BUILD_no_service_yaml.bazel") + self.assertEqual("", parsed.service_yaml) + + def test_gapic_inputs_parse_proto_only_returns_grpc(self): + parsed = parse(build_file, "test/versioned/path", "BUILD_proto_only.bazel") + self.assertEqual("grpc", parsed.transport) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_comment_common_resources.bazel b/hermetic_build/common/tests/resources/misc/BUILD_comment_common_resources.bazel new file mode 100644 index 0000000000..126ffdb7ca --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_comment_common_resources.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + #"//google/cloud:common_resources_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/misc/BUILD_comment_iam_policy.bazel b/hermetic_build/common/tests/resources/misc/BUILD_comment_iam_policy.bazel new file mode 100644 index 0000000000..a9a2c1ca75 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_comment_iam_policy.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + # "//google/iam/v1:iam_policy_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/misc/BUILD_comment_locations.bazel b/hermetic_build/common/tests/resources/misc/BUILD_comment_locations.bazel new file mode 100644 index 0000000000..8b96e3ab81 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_comment_locations.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + # "//google/cloud/location:location_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/misc/BUILD_common_resources.bazel b/hermetic_build/common/tests/resources/misc/BUILD_common_resources.bazel new file mode 100644 index 0000000000..9b749e6ad5 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_common_resources.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + "//google/cloud:common_resources_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/misc/BUILD_gapic_yaml.bazel b/hermetic_build/common/tests/resources/misc/BUILD_gapic_yaml.bazel new file mode 100644 index 0000000000..b55f4550d8 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_gapic_yaml.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + gapic_yaml = "test_gapic_yaml.yaml", +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_grpc.bazel b/hermetic_build/common/tests/resources/misc/BUILD_grpc.bazel new file mode 100644 index 0000000000..3e59a9bd35 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_grpc.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_transport_from_BUILD` in utilities.sh + +java_gapic_library( + transport = "grpc", +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_grpc_rest.bazel b/hermetic_build/common/tests/resources/misc/BUILD_grpc_rest.bazel new file mode 100644 index 0000000000..8a98e7e646 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_grpc_rest.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_transport_from_BUILD` in utilities.sh + +java_gapic_library( + transport = "grpc+rest", +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_iam_locations.bazel b/hermetic_build/common/tests/resources/misc/BUILD_iam_locations.bazel new file mode 100644 index 0000000000..d0c971da7c --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_iam_locations.bazel @@ -0,0 +1,6 @@ +proto_library_with_info( + deps = [ + "//google/cloud/location:location_proto", + "//google/iam/v1:iam_policy_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/misc/BUILD_iam_policy.bazel b/hermetic_build/common/tests/resources/misc/BUILD_iam_policy.bazel new file mode 100644 index 0000000000..af5d4a32f8 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_iam_policy.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + "//google/iam/v1:iam_policy_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/misc/BUILD_include_samples_empty.bazel b/hermetic_build/common/tests/resources/misc/BUILD_include_samples_empty.bazel new file mode 100644 index 0000000000..b9d4cd56e0 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_include_samples_empty.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh + +java_gapic_assembly_gradle_pkg( + +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_include_samples_false.bazel b/hermetic_build/common/tests/resources/misc/BUILD_include_samples_false.bazel new file mode 100644 index 0000000000..8e95c3d2a6 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_include_samples_false.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh + +java_gapic_assembly_gradle_pkg( + include_samples = False, +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_include_samples_true.bazel b/hermetic_build/common/tests/resources/misc/BUILD_include_samples_true.bazel new file mode 100644 index 0000000000..bac72b678f --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_include_samples_true.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh + +java_gapic_assembly_gradle_pkg( + include_samples = True, +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_locations.bazel b/hermetic_build/common/tests/resources/misc/BUILD_locations.bazel new file mode 100644 index 0000000000..29ee14fdba --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_locations.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + "//google/cloud/location:location_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/misc/BUILD_no_additional_protos.bazel b/hermetic_build/common/tests/resources/misc/BUILD_no_additional_protos.bazel new file mode 100644 index 0000000000..a22257cad4 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_no_additional_protos.bazel @@ -0,0 +1,4 @@ +proto_library_with_info( + deps = [ + ] +) \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/misc/BUILD_no_gapic_yaml.bazel b/hermetic_build/common/tests/resources/misc/BUILD_no_gapic_yaml.bazel new file mode 100644 index 0000000000..1e9462aa30 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_no_gapic_yaml.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + gapic_yaml = None +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_no_service_config.bazel b/hermetic_build/common/tests/resources/misc/BUILD_no_service_config.bazel new file mode 100644 index 0000000000..dbde6de05c --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_no_service_config.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + grpc_service_config = None +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_no_service_yaml.bazel b/hermetic_build/common/tests/resources/misc/BUILD_no_service_yaml.bazel new file mode 100644 index 0000000000..05bae16d5d --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_no_service_yaml.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + service_yaml = None +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_proto_only.bazel b/hermetic_build/common/tests/resources/misc/BUILD_proto_only.bazel new file mode 100644 index 0000000000..26bcea6126 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_proto_only.bazel @@ -0,0 +1,16 @@ +java_gapic_assembly_gradle_pkg( + name = "google-api-java", + transport = "grpc+rest", + deps = [ + "annotations_proto", + "auth_proto", + "backend_proto", + "billing_proto", + "client_proto", + "config_change_proto", + "consumer_proto", + "context_proto", + "control_proto", + "distribution_proto", + ], +) \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/misc/BUILD_rest.bazel b/hermetic_build/common/tests/resources/misc/BUILD_rest.bazel new file mode 100644 index 0000000000..9dff694297 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_rest.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_transport_from_BUILD` in utilities.sh + +java_gapic_library( + transport = "rest", +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel b/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel new file mode 100644 index 0000000000..992b91e52c --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh + +java_gapic_library( + +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel b/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel new file mode 100644 index 0000000000..a446c6b2a7 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh + +java_gapic_library( + rest_numeric_enums = False +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel b/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel new file mode 100644 index 0000000000..c4d7fefeb4 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh + +java_gapic_library( + rest_numeric_enums = True +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_service_config.bazel b/hermetic_build/common/tests/resources/misc/BUILD_service_config.bazel new file mode 100644 index 0000000000..097d1bb6bd --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_service_config.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + grpc_service_config = "test_service_config.json" +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_service_config_relative_target.bazel b/hermetic_build/common/tests/resources/misc/BUILD_service_config_relative_target.bazel new file mode 100644 index 0000000000..ccd59af2fb --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_service_config_relative_target.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + grpc_service_config = ":compute_grpc_service_config.json" +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_service_yaml.bazel b/hermetic_build/common/tests/resources/misc/BUILD_service_yaml.bazel new file mode 100644 index 0000000000..f7e4c91f4e --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_service_yaml.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + service_yaml = "test_service_yaml.yaml" +) diff --git a/hermetic_build/common/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel b/hermetic_build/common/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel new file mode 100644 index 0000000000..ded899dff7 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + service_yaml = "//google/cloud/videointelligence:videointelligence_v1p3beta1.yaml", +) \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/misc/TESTWORKSPACE b/hermetic_build/common/tests/resources/misc/TESTWORKSPACE new file mode 100644 index 0000000000..60b3036d9d --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/TESTWORKSPACE @@ -0,0 +1,133 @@ +# test workspace file obtained from sdk-platform-java + +workspace(name = "gapic_generator_java") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# gax-java and its transitive dependencies must be imported before +# gapic-generator-java dependencies to match the order in googleapis repository, +# which in its turn, prioritizes actual generated clients runtime dependencies +# over the generator dependencies. +local_repository( + name = "com_google_api_gax_java", + path = "gax-java", +) + +load("@com_google_api_gax_java//:repository_rules.bzl", "com_google_api_gax_java_properties") + +com_google_api_gax_java_properties( + name = "com_google_api_gax_java_properties", + file = "@com_google_api_gax_java//:dependencies.properties", +) + +load("@com_google_api_gax_java//:repositories.bzl", "com_google_api_gax_java_repositories") + +com_google_api_gax_java_repositories() + +_googleapis_commit = "7438480b2a1bc6371d748e974f7a3647f90c4e8d" + +http_archive( + name = "com_google_googleapis", + strip_prefix = "googleapis-%s" % _googleapis_commit, + urls = [ + "https://github.com/googleapis/googleapis/archive/%s.zip" % _googleapis_commit, + ], +) + +# protobuf +RULES_JVM_EXTERNAL_TAG = "4.5" + +RULES_JVM_EXTERNAL_SHA = "b17d7388feb9bfa7f2fa09031b32707df529f26c91ab9e5d909eb1676badd9a6" + +http_archive( + name = "rules_jvm_external", + sha256 = RULES_JVM_EXTERNAL_SHA, + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, + url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, +) + +load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") + +rules_jvm_external_deps() + +load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") + +rules_jvm_external_setup() + +load("@com_google_protobuf//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") +load("@rules_jvm_external//:defs.bzl", "maven_install") + +maven_install( + artifacts = PROTOBUF_MAVEN_ARTIFACTS, + repositories = ["https://repo.maven.apache.org/maven2/"], +) + +_gapic_generator_java_version = "2.25.1-SNAPSHOT" # {x-version-update:gapic-generator-java:current} + +maven_install( + artifacts = [ + "com.google.api:gapic-generator-java:" + _gapic_generator_java_version, + ], + fail_on_missing_checksum = False, + repositories = [ + "m2Local", + "https://repo.maven.apache.org/maven2/", + ], +) + +protobuf_deps() + +# Bazel rules. +_rules_gapic_version = "0.5.5" + +http_archive( + name = "rules_gapic", + strip_prefix = "rules_gapic-%s" % _rules_gapic_version, + urls = ["https://github.com/googleapis/rules_gapic/archive/v%s.tar.gz" % _rules_gapic_version], +) + +# Java dependencies. +load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") + +switched_rules_by_language( + name = "com_google_googleapis_imports", + gapic = True, + grpc = True, + java = True, +) + +load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories") + +grpc_java_repositories() + +_disco_to_proto3_converter_commit = "ce8d8732120cdfb5bf4847c3238b5be8acde87e3" + +http_archive( + name = "com_google_disco_to_proto3_converter", + strip_prefix = "disco-to-proto3-converter-%s" % _disco_to_proto3_converter_commit, + urls = ["https://github.com/googleapis/disco-to-proto3-converter/archive/%s.zip" % _disco_to_proto3_converter_commit], +) + +# Showcase +_showcase_version = "0.28.2" + +http_archive( + name = "com_google_gapic_showcase", + strip_prefix = "gapic-showcase-%s" % _showcase_version, + urls = [ + "https://github.com/googleapis/gapic-showcase/archive/refs/tags/v%s.zip" % _showcase_version, + ], +) + +http_archive( + name = "rules_pkg", + sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", + ], +) + +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") + +rules_pkg_dependencies() diff --git a/hermetic_build/common/tests/resources/misc/versions.txt b/hermetic_build/common/tests/resources/misc/versions.txt new file mode 100644 index 0000000000..e4258504e4 --- /dev/null +++ b/hermetic_build/common/tests/resources/misc/versions.txt @@ -0,0 +1,9 @@ +# test versions.txt obtained from sdk-platform-java + +gapic-generator-java:2.25.0:2.25.1-SNAPSHOT +api-common:2.16.0:2.16.1-SNAPSHOT +gax:2.33.0:2.33.1-SNAPSHOT +gax-grpc:2.34.0:2.33.1-SNAPSHOT +gax-httpjson:0.118.0:0.118.1-SNAPSHOT +proto-google-common-protos:2.24.0:2.24.1-SNAPSHOT +grpc-google-common-protos:2.24.0:2.24.1-SNAPSHOT diff --git a/hermetic_build/release_note_generation/setup.py b/hermetic_build/release_note_generation/setup.py new file mode 100755 index 0000000000..1adf08151a --- /dev/null +++ b/hermetic_build/release_note_generation/setup.py @@ -0,0 +1,17 @@ +""" +Package information of library_generation python scripts +""" + +from setuptools import setup + +setup( + name="common", + version="0.1", + python_requires=">=3.12", + package_dir={ + "common": ".", + }, + install_requires=[ + "PyYAML==6.0.2", + ], +) diff --git a/hermetic_build/release_note_generation/tests/__init__.py b/hermetic_build/release_note_generation/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/release_note_generation/tests/cli/__init__.py b/hermetic_build/release_note_generation/tests/cli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py b/hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py new file mode 100644 index 0000000000..be0d6a4fed --- /dev/null +++ b/hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py @@ -0,0 +1,71 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +from click.testing import CliRunner +from library_generation.cli.generate_release_note import generate + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resource_dir = os.path.join(script_dir, "..", "resources", "test_generate_release_note") + + +class GenerateReleaseNoteTest(unittest.TestCase): + def test_gen_release_note_with_no_baseline_config_does_not_generate_note(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke(generate) + self.assertEqual(0, result.exit_code) + self.assertFalse(os.path.isfile("pr_description.txt")) + + def test_gen_release_note_with_no_current_config_does_not_generate_note(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + generate, ["--baseline-generation-config-path=any_config"] + ) + self.assertEqual(0, result.exit_code) + self.assertFalse(os.path.isfile("pr_description.txt")) + + def test_gen_release_note_with_nonexistent_baseline_config_does_not_generate_note( + self, + ): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + generate, + [ + "--baseline-generation-config-path=non_existent_config", + "--current-generation-config-path=not_relevant", + ], + ) + self.assertEqual(0, result.exit_code) + self.assertFalse(os.path.isfile("pr_description.txt")) + + def test_gen_release_note_with_nonexistent_current_config_does_not_generate_note( + self, + ): + cwd = os.getcwd() + os.chdir(resource_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + generate, + [ + "--baseline-generation-config-path=empty_gen_config.yaml", + "--current-generation-config-path=non_existent_config", + ], + ) + self.assertEqual(0, result.exit_code) + self.assertFalse(os.path.isfile("pr_description.txt")) + os.chdir(cwd) diff --git a/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py b/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py new file mode 100644 index 0000000000..16e3fffdfc --- /dev/null +++ b/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py @@ -0,0 +1,199 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from unittest.mock import patch + +from library_generation.model.config_change import ( + ConfigChange, + ChangeType, + LibraryChange, +) +from library_generation.model.generation_config import GenerationConfig +from library_generation.utils.commit_message_formatter import ( + format_commit_message, + commit_link, + format_repo_level_change, +) +from library_generation.utils.commit_message_formatter import wrap_googleapis_commit +from library_generation.utils.commit_message_formatter import wrap_override_commit + +gen_config = GenerationConfig( + gapic_generator_version="1.2.3", googleapis_commitish="123abc", libraries=[] +) + + +class CommitMessageFormatterTest(unittest.TestCase): + def test_format_commit_message_should_add_library_name_for_conventional_commit( + self, + ): + with patch("git.Commit") as mock_commit: + commit = mock_commit.return_value + commit.message = "feat: a commit message\nPiperOrigin-RevId: 123456" + commit.hexsha = "1234567abcdefg" + commits = {commit: "example_library"} + self.assertEqual( + [ + "BEGIN_NESTED_COMMIT", + "feat: [example_library] a commit message", + "PiperOrigin-RevId: 123456", + "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", + "END_NESTED_COMMIT", + ], + format_commit_message(commits, True), + ) + + def test_format_commit_message_should_add_library_name_for_mutliline_conventional_commit( + self, + ): + with patch("git.Commit") as mock_commit: + commit = mock_commit.return_value + commit.message = "feat: a commit message\nfix: an another commit message\nPiperOrigin-RevId: 123456" + commit.hexsha = "1234567abcdefg" + commits = {commit: "example_library"} + self.assertEqual( + [ + "BEGIN_NESTED_COMMIT", + "feat: [example_library] a commit message", + "fix: [example_library] an another commit message", + "PiperOrigin-RevId: 123456", + "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", + "END_NESTED_COMMIT", + ], + format_commit_message(commits, True), + ) + + def test_format_commit_message_should_not_add_library_name_for_nonconvnentional_commit( + self, + ): + with patch("git.Commit") as mock_commit: + commit = mock_commit.return_value + commit.message = "PiperOrigin-RevId: 123456" + commit.hexsha = "1234567abcdefg" + commits = {commit: "example_library"} + self.assertEqual( + [ + "BEGIN_NESTED_COMMIT", + "PiperOrigin-RevId: 123456", + "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", + "END_NESTED_COMMIT", + ], + format_commit_message(commits, True), + ) + + def test_format_commit_message_should_not_add_library_name_if_not_monorepo(self): + with patch("git.Commit") as mock_commit: + commit = mock_commit.return_value + commit.message = "feat: a commit message\nPiperOrigin-RevId: 123456" + commit.hexsha = "1234567abcdefg" + commits = {commit: "example_library"} + self.assertEqual( + [ + "BEGIN_NESTED_COMMIT", + "feat: a commit message", + "PiperOrigin-RevId: 123456", + "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", + "END_NESTED_COMMIT", + ], + format_commit_message(commits, False), + ) + + def test_format_commit_message_should_not_add_library_name_for_multiline_commit_if_not_monorepo( + self, + ): + with patch("git.Commit") as mock_commit: + commit = mock_commit.return_value + commit.message = "feat: a commit message\nfix: an another commit message\nPiperOrigin-RevId: 123456" + commit.hexsha = "1234567abcdefg" + commits = {commit: "example_library"} + self.assertEqual( + [ + "BEGIN_NESTED_COMMIT", + "feat: a commit message", + "fix: an another commit message", + "PiperOrigin-RevId: 123456", + "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", + "END_NESTED_COMMIT", + ], + format_commit_message(commits, False), + ) + + def test_wrap_nested_commit_success(self): + with patch("git.Commit") as mock_commit: + commit = mock_commit.return_value + commit.hexsha = "1234567abcdefg" + messages = ["a commit message", "another message"] + self.assertEqual( + [ + "BEGIN_NESTED_COMMIT", + "a commit message", + "another message", + "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", + "END_NESTED_COMMIT", + ], + wrap_googleapis_commit(commit, messages), + ) + + def test_wrap_override_commit_success(self): + messages = ["a commit message", "another message"] + self.assertEqual( + [ + "BEGIN_COMMIT_OVERRIDE", + "a commit message", + "another message", + "END_COMMIT_OVERRIDE", + ], + wrap_override_commit(messages), + ) + + def test_commit_link_success(self): + with patch("git.Commit") as mock_commit: + commit = mock_commit.return_value + commit.hexsha = "1234567abcdefg" + self.assertEqual( + "[googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", + commit_link(commit), + ) + + def test_format_repo_level_change_success(self): + config_change = ConfigChange( + change_to_libraries={ + ChangeType.REPO_LEVEL_CHANGE: [ + LibraryChange( + changed_param="gapic_generator_version", current_value="1.2.3" + ), + LibraryChange( + changed_param="libraries_bom_version", current_value="2.3.4" + ), + LibraryChange( + changed_param="protoc_version", current_value="3.4.5" + ), + ] + }, + baseline_config=gen_config, + current_config=gen_config, + ) + self.assertEqual( + [ + "BEGIN_NESTED_COMMIT", + "fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3", + "END_NESTED_COMMIT", + "BEGIN_NESTED_COMMIT", + "chore: update the libraries_bom version to 2.3.4", + "END_NESTED_COMMIT", + "BEGIN_NESTED_COMMIT", + "chore: update repo-level parameter protoc_version to 3.4.5", + "END_NESTED_COMMIT", + ], + format_repo_level_change(config_change), + ) diff --git a/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py b/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py new file mode 100644 index 0000000000..9adbe71277 --- /dev/null +++ b/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py @@ -0,0 +1,270 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +from filecmp import cmp + +from library_generation.generate_pr_description import ( + get_repo_level_commit_messages, + generate_pr_descriptions, +) +from library_generation.model.config_change import ( + ConfigChange, + ChangeType, + LibraryChange, +) +from library_generation.model.gapic_config import GapicConfig +from library_generation.model.generation_config import GenerationConfig +from library_generation.model.library_config import LibraryConfig + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "resources", "goldens") + + +class GeneratePrDescriptionTest(unittest.TestCase): + def test_get_commit_messages_current_is_older_raise_exception(self): + # committed on April 1st, 2024 + current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + # committed on April 2nd, 2024 + baseline_commit = "d5020fff4cbe108bdf506074791c56cff7840bef" + self.assertRaisesRegex( + ValueError, + "newer than", + get_repo_level_commit_messages, + "https://github.com/googleapis/googleapis.git", + current_commit, + baseline_commit, + {}, + True, + [], + ) + + def test_get_commit_messages_with_same_current_and_baseline_returns_empty_message( + self, + ): + # committed on April 1st, 2024 + current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + baseline_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + self.assertEqual( + "", + get_repo_level_commit_messages( + "https://github.com/googleapis/googleapis.git", + current_commit, + baseline_commit, + {}, + True, + [], + ), + ) + + def test_generate_pr_description_with_no_change_in_config(self): + commit_sha = "36441693dddaf0ed73951ad3a15c215a332756aa" + config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish=commit_sha, + libraries_bom_version="", + # use empty libraries to make sure no qualified commit between + # two commit sha. + libraries=[], + ) + pr_description_path = os.path.join(os.getcwd(), "no_config_change") + generate_pr_descriptions( + config_change=ConfigChange( + change_to_libraries={}, + baseline_config=config, + current_config=config, + ), + description_path=pr_description_path, + ) + self.assertFalse(os.path.isfile(f"{pr_description_path}/pr_description.txt")) + + def test_generate_pr_description_does_not_create_pr_description_without_qualified_commit( + self, + ): + # committed on May 22nd, 2024 + old_commit_sha = "30717c0b0c9966906880703208a4c820411565c4" + # committed on May 23rd, 2024 + new_commit_sha = "eeed69d446a90eb4a4a2d1762c49d637075390c1" + pr_description_path = os.path.join(os.getcwd(), "no_qualified_commit") + generate_pr_descriptions( + config_change=ConfigChange( + change_to_libraries={}, + baseline_config=GenerationConfig( + gapic_generator_version="", + googleapis_commitish=old_commit_sha, + # use empty libraries to make sure no qualified commit between + # two commit sha. + libraries=[], + ), + current_config=GenerationConfig( + gapic_generator_version="", + googleapis_commitish=new_commit_sha, + # use empty libraries to make sure no qualified commit between + # two commit sha. + libraries=[], + ), + ), + description_path=pr_description_path, + ) + self.assertFalse(os.path.isfile(f"{pr_description_path}/pr_description.txt")) + + def test_generate_pr_description_with_combined_message( + self, + ): + # no other commits between these two commits. + baseline_commit_sha = "3b6f144d47b0a1d2115ab2445ec06e80cc324a44" + documentai_commit_sha = "0cea7170404bec3d994f43db4fa292f5034cbe9a" + cwd = os.getcwd() + library = LibraryConfig( + api_shortname="documentai", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[GapicConfig(proto_path="google/cloud/documentai/v1")], + ) + generate_pr_descriptions( + config_change=ConfigChange( + change_to_libraries={ + ChangeType.REPO_LEVEL_CHANGE: [ + LibraryChange( + changed_param="gapic_generator_version", + current_value="1.2.3", + ), + LibraryChange( + changed_param="libraries_bom_version", current_value="2.3.4" + ), + ], + ChangeType.GOOGLEAPIS_COMMIT: [], + }, + baseline_config=GenerationConfig( + gapic_generator_version="", + googleapis_commitish=baseline_commit_sha, + libraries=[library], + ), + current_config=GenerationConfig( + gapic_generator_version="1.2.3", + googleapis_commitish=documentai_commit_sha, + libraries_bom_version="2.3.4", + libraries=[library], + ), + ), + description_path=cwd, + ) + self.assertTrue(os.path.isfile(f"{cwd}/pr_description.txt")) + self.assertTrue( + cmp( + f"{resources_dir}/pr_description-golden.txt", + f"{cwd}/pr_description.txt", + ), + "The generated PR description does not match the expected golden file", + ) + os.remove(f"{cwd}/pr_description.txt") + + def test_generate_pr_description_with_repo_level_change_without_qualified_commit( + self, + ): + # no other commits between these two commits. + baseline_commit_sha = "3b6f144d47b0a1d2115ab2445ec06e80cc324a44" + current_commit_sha = "0cea7170404bec3d994f43db4fa292f5034cbe9a" + cwd = os.getcwd() + library = LibraryConfig( + api_shortname="example_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[GapicConfig(proto_path="google/example/v1")], + ) + generate_pr_descriptions( + config_change=ConfigChange( + change_to_libraries={ + ChangeType.REPO_LEVEL_CHANGE: [ + LibraryChange( + changed_param="gapic_generator_version", + current_value="1.2.3", + ), + LibraryChange( + changed_param="libraries_bom_version", current_value="2.3.4" + ), + ], + ChangeType.GOOGLEAPIS_COMMIT: [], + }, + baseline_config=GenerationConfig( + gapic_generator_version="", + googleapis_commitish=baseline_commit_sha, + libraries=[library], + ), + current_config=GenerationConfig( + gapic_generator_version="1.2.3", + googleapis_commitish=current_commit_sha, + libraries_bom_version="2.3.4", + libraries=[library], + ), + ), + description_path=cwd, + ) + self.assertTrue(os.path.isfile(f"{cwd}/pr_description.txt")) + self.assertTrue( + cmp( + f"{resources_dir}/repo_level_and_no_qualified_commit_pr_description-golden.txt", + f"{cwd}/pr_description.txt", + ), + "The generated PR description does not match the expected golden file", + ) + os.remove(f"{cwd}/pr_description.txt") + + def test_generate_pr_description_create_description_with_only_repo_level_change( + self, + ): + commit_sha = "3b6f144d47b0a1d2115ab2445ec06e80cc324a44" + cwd = os.getcwd() + library = LibraryConfig( + api_shortname="documentai", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[GapicConfig(proto_path="google/cloud/documentai/v1")], + ) + generate_pr_descriptions( + config_change=ConfigChange( + change_to_libraries={ + ChangeType.REPO_LEVEL_CHANGE: [ + LibraryChange( + changed_param="gapic_generator_version", + current_value="1.2.3", + ) + ], + ChangeType.GOOGLEAPIS_COMMIT: [], + }, + baseline_config=GenerationConfig( + gapic_generator_version="1.2.2", + googleapis_commitish=commit_sha, + libraries=[library], + ), + current_config=GenerationConfig( + gapic_generator_version="1.2.3", + googleapis_commitish=commit_sha, + libraries=[library], + ), + ), + description_path=cwd, + ) + self.assertTrue(os.path.isfile(f"{cwd}/pr_description.txt")) + self.assertTrue( + cmp( + f"{resources_dir}/repo_level_only_pr_description-golden.txt", + f"{cwd}/pr_description.txt", + ), + "The generated PR description does not match the expected golden file", + ) + os.remove(f"{cwd}/pr_description.txt") From 40c12afe3dde963047d07e3fd8c62803877b4701 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 12:24:20 -0400 Subject: [PATCH 16/69] remove library_generation dir --- .../library_generation/DEVELOPMENT.md | 207 ++++++ hermetic_build/library_generation/README.md | 310 +++++++++ library_generation/__init__.py | 0 library_generation/cli/entry_point.py | 244 ------- .../cli/generate_monorepo_gapic_bom.py | 49 -- .../cli/generate_monorepo_root_pom.py | 39 -- .../cli/generate_release_note.py | 98 --- library_generation/dockerignore | 6 - .../gapic-generator-java-wrapper | 7 - .../generate_composed_library.py | 173 ----- library_generation/generate_library.sh | 278 -------- library_generation/generate_pr_description.py | 191 ------ library_generation/generate_repo.py | 88 --- library_generation/model/bom_config.py | 45 -- library_generation/model/config_change.py | 203 ------ library_generation/model/gapic_config.py | 99 --- library_generation/model/gapic_inputs.py | 196 ------ library_generation/model/generation_config.py | 198 ------ library_generation/model/library_config.py | 203 ------ library_generation/model/repo_config.py | 80 --- library_generation/owlbot/bin/entrypoint.sh | 78 --- .../owlbot/bin/format_source.sh | 47 -- .../owlbot/bin/write_clirr_ignore.sh | 34 - .../owlbot/src/fix-license-headers.py | 30 - library_generation/owlbot/src/fix_poms.py | 611 ------------------ library_generation/owlbot/src/gen-template.py | 84 --- library_generation/owlbot/src/poms/.gitignore | 1 - library_generation/owlbot/src/poms/module.py | 54 -- .../owlbot/src/poms/templates.py | 36 -- .../owlbot/synthtool/__init__.py | 32 - .../owlbot/synthtool/_tracked_paths.py | 39 -- .../owlbot/synthtool/gcp/common.py | 146 ----- .../owlbot/synthtool/gcp/samples.py | 91 --- .../owlbot/synthtool/gcp/snippets.py | 124 ---- .../owlbot/synthtool/languages/java.py | 611 ------------------ .../owlbot/synthtool/sources/templates.py | 81 --- .../owlbot/synthtool/transforms.py | 314 --------- .../clirr/clirr-ignored-differences.xml.j2 | 80 --- .../templates/java_library/.github/CODEOWNERS | 20 - .../.github/ISSUE_TEMPLATE/bug_report.md | 56 -- .../.github/ISSUE_TEMPLATE/feature_request.md | 26 - .../.github/ISSUE_TEMPLATE/support_request.md | 7 - .../.github/PULL_REQUEST_TEMPLATE.md | 10 - .../java_library/.github/auto-label.yaml | 15 - .../java_library/.github/blunderbuss.yml | 7 - .../java_library/.github/dependabot.yml | 19 - .../java_library/.github/release-please.yml | 3 - .../java_library/.github/release-trigger.yml | 2 - .../scripts/update_generation_config.sh | 121 ---- .../java_library/.github/snippet-bot.yml | 0 .../.github/sync-repo-settings.yaml | 64 -- .../.github/trusted-contribution.yml | 9 - .../.github/workflows/approve-readme.yaml | 69 -- .../java_library/.github/workflows/ci.yaml | 123 ---- .../workflows/renovate_config_check.yaml | 25 - .../.github/workflows/samples.yaml | 30 - .../workflows/update_generation_config.yaml | 43 -- .../templates/java_library/.kokoro/build.sh | 28 - .../templates/java_library/.kokoro/common.cfg | 19 - .../java_library/.kokoro/continuous.cfg | 1 - .../java_library/.kokoro/presubmit.cfg | 1 - .../java_library/.kokoro/trampoline.sh | 24 - .../templates/java_library/CODE_OF_CONDUCT.md | 94 --- .../templates/java_library/CONTRIBUTING.md | 92 --- .../owlbot/templates/java_library/LICENSE | 201 ------ .../owlbot/templates/java_library/README.md | 288 --------- .../owlbot/templates/java_library/SECURITY.md | 7 - .../owlbot/templates/java_library/java.header | 15 - .../templates/java_library/license-checks.xml | 10 - .../templates/java_library/renovate.json | 128 ---- .../samples/install-without-bom/pom.xml | 86 --- .../templates/java_library/samples/pom.xml | 56 -- .../java_library/samples/snapshot/pom.xml | 85 --- .../java_library/samples/snippets/pom.xml | 49 -- .../owlbot/templates/poms/bom_pom.xml.j2 | 41 -- .../owlbot/templates/poms/cloud_pom.xml.j2 | 156 ----- .../owlbot/templates/poms/grpc_pom.xml.j2 | 71 -- .../owlbot/templates/poms/parent_pom.xml.j2 | 51 -- .../owlbot/templates/poms/proto_pom.xml.j2 | 48 -- .../owlbot/templates/poms/versions.txt.j2 | 4 - library_generation/postprocess_library.sh | 111 ---- library_generation/requirements.in | 29 - library_generation/requirements.txt | 546 ---------------- library_generation/setup.py | 31 - .../templates/gapic-libraries-bom.xml.j2 | 37 -- library_generation/templates/owlbot.py.j2 | 28 - .../templates/owlbot.yaml.monorepo.j2 | 36 -- library_generation/templates/root-pom.xml.j2 | 88 --- library_generation/test/__init__.py | 0 library_generation/test/cli/__init__.py | 0 .../test/cli/entry_point_unit_tests.py | 416 ------------ .../cli/generate_release_note_unit_tests.py | 71 -- library_generation/test/compare_poms.py | 131 ---- .../test/generate_library_unit_tests.py | 109 ---- .../test/generate_library_unit_tests.sh | 314 --------- .../generate_pr_description_unit_tests.py | 270 -------- .../test/generate_repo_unit_tests.py | 73 --- library_generation/test/integration_tests.py | 394 ----------- library_generation/test/model/__init__.py | 0 .../test/model/config_change_unit_tests.py | 314 --------- .../test/model/gapic_config_unit_tests.py | 122 ---- .../test/model/gapic_inputs_unit_tests.py | 132 ---- .../model/generation_config_unit_tests.py | 259 -------- .../test/model/library_config_unit_tests.py | 122 ---- .../test/model/repo_config_unit_tests.py | 56 -- library_generation/test/owlbot/__init__.py | 0 .../test/owlbot/fix_poms_unit_tests.py | 47 -- .../test/owlbot/java_unit_tests.py | 310 --------- library_generation/test/owlbot/util.py | 126 ---- .../gapic_options/QueryServiceGrpc_copy.java | 9 - .../test/resources/gapic_options/example.yaml | 1 - .../gapic_options/example_gapic.legacy.yaml | 1 - .../gapic_options/example_gapic.yaml | 1 - .../gapic_options/example_gapic_legacy.yaml | 1 - .../example_grpc_service_config.json | 3 - .../goldens/.OwlBot-hermetic-golden.yaml | 35 - ...repo-metadata-custom-transport-golden.json | 16 - .../.repo-metadata-monorepo-golden.json | 20 - .../.repo-metadata-non-monorepo-golden.json | 21 - .../.repo-metadata-proto-only-golden.json | 19 - .../test/resources/goldens/owlbot-golden.py | 36 -- .../goldens/pr_description-golden.txt | 17 - ...qualified_commit_pr_description-golden.txt | 10 - .../repo_level_only_pr_description-golden.txt | 5 - .../baseline_generation_config.yaml | 45 -- .../current_generation_config.yaml | 45 -- .../pr-description-golden.txt | 54 -- .../java-bigtable/generation_config.yaml | 22 - .../java-bigtable/pr-description-golden.txt | 18 - .../integration/test_generator_coordinates | 1 - .../misc/BUILD_comment_common_resources.bazel | 5 - .../misc/BUILD_comment_iam_policy.bazel | 5 - .../misc/BUILD_comment_locations.bazel | 5 - .../misc/BUILD_common_resources.bazel | 5 - .../resources/misc/BUILD_gapic_yaml.bazel | 3 - .../test/resources/misc/BUILD_grpc.bazel | 5 - .../test/resources/misc/BUILD_grpc_rest.bazel | 5 - .../resources/misc/BUILD_iam_locations.bazel | 6 - .../resources/misc/BUILD_iam_policy.bazel | 5 - .../misc/BUILD_include_samples_empty.bazel | 5 - .../misc/BUILD_include_samples_false.bazel | 5 - .../misc/BUILD_include_samples_true.bazel | 5 - .../test/resources/misc/BUILD_locations.bazel | 5 - .../misc/BUILD_no_additional_protos.bazel | 4 - .../resources/misc/BUILD_no_gapic_yaml.bazel | 3 - .../misc/BUILD_no_service_config.bazel | 3 - .../misc/BUILD_no_service_yaml.bazel | 3 - .../resources/misc/BUILD_proto_only.bazel | 16 - .../test/resources/misc/BUILD_rest.bazel | 5 - .../misc/BUILD_rest_numeric_enums_empty.bazel | 5 - .../misc/BUILD_rest_numeric_enums_false.bazel | 5 - .../misc/BUILD_rest_numeric_enums_true.bazel | 5 - .../resources/misc/BUILD_service_config.bazel | 3 - ...BUILD_service_config_relative_target.bazel | 3 - .../resources/misc/BUILD_service_yaml.bazel | 3 - .../BUILD_service_yaml_absolute_target.bazel | 3 - .../test/resources/misc/TESTWORKSPACE | 133 ---- .../test/resources/misc/versions.txt | 9 - .../proto/google/cloud/test/v1/empty.proto | 0 .../proto-1/fake.proto | 0 .../proto-2/fake.proto | 0 .../config_without_api_description.yaml | 5 - .../config_without_api_shortname.yaml | 4 - .../config_without_gapics_key.yaml | 3 - .../config_without_gapics_value.yaml | 4 - .../config_without_googleapis.yaml | 8 - .../test-config/config_without_libraries.yaml | 1 - .../config_without_library_value.yaml | 2 - .../config_without_name_pretty.yaml | 6 - .../config_without_product_docs.yaml | 7 - .../config_without_proto_path.yaml | 4 - .../config_without_temp_excludes.yaml | 10 - .../test-config/generation_config.yaml | 24 - .../generation_config_library_modified.yaml | 14 - ...on_config_with_duplicate_library_name.yaml | 51 -- .../test-config/monorepo_baseline.yaml | 30 - .../test-config/monorepo_current.yaml | 30 - .../monorepo_with_common_protos.yaml | 43 -- .../monorepo_without_common_protos.yaml | 33 - .../test-monorepo/.github/.OwlBot.lock.yaml | 17 - .../test-service/.repo-metadata.json | 18 - .../.github/release-please.yml | 6 - .../release-please-update/.repo-metadata.json | 18 - .../render-readme/.repo-metadata.json | 18 - .../standard/.repo-metadata.json | 18 - .../java-admanager/.repo-metadata.json | 16 - .../ad-manager-bom/pom-golden.xml | 38 -- .../java-admanager/ad-manager/pom-golden.xml | 110 ---- .../test-owlbot/java-admanager/pom-golden.xml | 49 -- .../proto-ad-manager-v1/pom-golden.xml | 37 -- .../test-owlbot/java-admanager/versions.txt | 6 - .../test-owlbot/testdata/FooGrpcGolden.java | 22 - .../test-owlbot/testdata/FooProtoGolden.java | 25 - .../test-owlbot/testdata/README-golden.md | 202 ------ .../test-owlbot/testdata/SampleClass.java | 29 - .../testdata/SampleClassGolden.java | 23 - .../testdata/SampleCopyMethodGolden.java | 37 -- .../testdata/SampleDeprecateClass.java | 40 -- .../testdata/SampleDeprecateMethodGolden.java | 52 -- .../test-owlbot/testdata/src/foo/FooGrpc.java | 7 - .../testdata/src/foo/FooProto.java | 10 - .../empty_gen_config.yaml | 0 .../gapic-libraries-bom/pom-golden.xml | 45 -- .../java-dns/pom.xml | 9 - .../google-cloud-service-control-bom/pom.xml | 8 - .../java-tasks/google-cloud-tasks-bom/pom.xml | 8 - .../pom-golden.xml | 88 --- .../test_monorepo_postprocessing/versions.txt | 4 - .../src/main/java/example_main.txt | 0 .../src/test/java/example_test.txt | 0 .../src/main/java/example_proto_main.txt | 0 .../src/main/java/com/example_com_sample.txt | 0 .../src/main/java/io/example_io_sample.txt | 0 library_generation/test/test_utilities.sh | 129 ---- library_generation/test/test_utils.py | 40 -- .../test/utilities_unit_tests.py | 391 ----------- library_generation/test/utils/__init__.py | 0 .../commit_message_formatter_unit_tests.py | 199 ------ ...generation_config_comparator_unit_tests.py | 506 --------------- .../monorepo_postprocessor_unit_tests.py | 46 -- .../test/utils/pom_generator_unit_tests.py | 32 - .../test/utils/proto_path_utils_unit_tests.py | 50 -- library_generation/utils/__init__.py | 0 .../utils/commit_message_formatter.py | 127 ---- library_generation/utils/file_render.py | 24 - .../utils/generation_config_comparator.py | 284 -------- .../utils/monorepo_postprocessor.py | 29 - library_generation/utils/pom_generator.py | 157 ----- library_generation/utils/proto_path_utils.py | 47 -- library_generation/utils/utilities.py | 310 --------- library_generation/utils/utilities.sh | 391 ----------- 231 files changed, 517 insertions(+), 15714 deletions(-) create mode 100644 hermetic_build/library_generation/DEVELOPMENT.md create mode 100644 hermetic_build/library_generation/README.md delete mode 100644 library_generation/__init__.py delete mode 100644 library_generation/cli/entry_point.py delete mode 100644 library_generation/cli/generate_monorepo_gapic_bom.py delete mode 100644 library_generation/cli/generate_monorepo_root_pom.py delete mode 100644 library_generation/cli/generate_release_note.py delete mode 100644 library_generation/dockerignore delete mode 100755 library_generation/gapic-generator-java-wrapper delete mode 100755 library_generation/generate_composed_library.py delete mode 100755 library_generation/generate_library.sh delete mode 100755 library_generation/generate_pr_description.py delete mode 100755 library_generation/generate_repo.py delete mode 100644 library_generation/model/bom_config.py delete mode 100644 library_generation/model/config_change.py delete mode 100644 library_generation/model/gapic_config.py delete mode 100644 library_generation/model/gapic_inputs.py delete mode 100644 library_generation/model/generation_config.py delete mode 100644 library_generation/model/library_config.py delete mode 100644 library_generation/model/repo_config.py delete mode 100755 library_generation/owlbot/bin/entrypoint.sh delete mode 100755 library_generation/owlbot/bin/format_source.sh delete mode 100755 library_generation/owlbot/bin/write_clirr_ignore.sh delete mode 100644 library_generation/owlbot/src/fix-license-headers.py delete mode 100644 library_generation/owlbot/src/fix_poms.py delete mode 100644 library_generation/owlbot/src/gen-template.py delete mode 100644 library_generation/owlbot/src/poms/.gitignore delete mode 100644 library_generation/owlbot/src/poms/module.py delete mode 100644 library_generation/owlbot/src/poms/templates.py delete mode 100644 library_generation/owlbot/synthtool/__init__.py delete mode 100644 library_generation/owlbot/synthtool/_tracked_paths.py delete mode 100644 library_generation/owlbot/synthtool/gcp/common.py delete mode 100644 library_generation/owlbot/synthtool/gcp/samples.py delete mode 100644 library_generation/owlbot/synthtool/gcp/snippets.py delete mode 100644 library_generation/owlbot/synthtool/languages/java.py delete mode 100644 library_generation/owlbot/synthtool/sources/templates.py delete mode 100644 library_generation/owlbot/synthtool/transforms.py delete mode 100644 library_generation/owlbot/templates/clirr/clirr-ignored-differences.xml.j2 delete mode 100644 library_generation/owlbot/templates/java_library/.github/CODEOWNERS delete mode 100644 library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/support_request.md delete mode 100644 library_generation/owlbot/templates/java_library/.github/PULL_REQUEST_TEMPLATE.md delete mode 100644 library_generation/owlbot/templates/java_library/.github/auto-label.yaml delete mode 100644 library_generation/owlbot/templates/java_library/.github/blunderbuss.yml delete mode 100644 library_generation/owlbot/templates/java_library/.github/dependabot.yml delete mode 100644 library_generation/owlbot/templates/java_library/.github/release-please.yml delete mode 100644 library_generation/owlbot/templates/java_library/.github/release-trigger.yml delete mode 100644 library_generation/owlbot/templates/java_library/.github/scripts/update_generation_config.sh delete mode 100644 library_generation/owlbot/templates/java_library/.github/snippet-bot.yml delete mode 100644 library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml delete mode 100644 library_generation/owlbot/templates/java_library/.github/trusted-contribution.yml delete mode 100644 library_generation/owlbot/templates/java_library/.github/workflows/approve-readme.yaml delete mode 100644 library_generation/owlbot/templates/java_library/.github/workflows/ci.yaml delete mode 100644 library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml delete mode 100644 library_generation/owlbot/templates/java_library/.github/workflows/samples.yaml delete mode 100644 library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml delete mode 100755 library_generation/owlbot/templates/java_library/.kokoro/build.sh delete mode 100644 library_generation/owlbot/templates/java_library/.kokoro/common.cfg delete mode 100644 library_generation/owlbot/templates/java_library/.kokoro/continuous.cfg delete mode 100644 library_generation/owlbot/templates/java_library/.kokoro/presubmit.cfg delete mode 100755 library_generation/owlbot/templates/java_library/.kokoro/trampoline.sh delete mode 100644 library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md delete mode 100644 library_generation/owlbot/templates/java_library/CONTRIBUTING.md delete mode 100644 library_generation/owlbot/templates/java_library/LICENSE delete mode 100644 library_generation/owlbot/templates/java_library/README.md delete mode 100644 library_generation/owlbot/templates/java_library/SECURITY.md delete mode 100644 library_generation/owlbot/templates/java_library/java.header delete mode 100644 library_generation/owlbot/templates/java_library/license-checks.xml delete mode 100644 library_generation/owlbot/templates/java_library/renovate.json delete mode 100644 library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml delete mode 100644 library_generation/owlbot/templates/java_library/samples/pom.xml delete mode 100644 library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml delete mode 100644 library_generation/owlbot/templates/java_library/samples/snippets/pom.xml delete mode 100644 library_generation/owlbot/templates/poms/bom_pom.xml.j2 delete mode 100644 library_generation/owlbot/templates/poms/cloud_pom.xml.j2 delete mode 100644 library_generation/owlbot/templates/poms/grpc_pom.xml.j2 delete mode 100644 library_generation/owlbot/templates/poms/parent_pom.xml.j2 delete mode 100644 library_generation/owlbot/templates/poms/proto_pom.xml.j2 delete mode 100644 library_generation/owlbot/templates/poms/versions.txt.j2 delete mode 100755 library_generation/postprocess_library.sh delete mode 100644 library_generation/requirements.in delete mode 100644 library_generation/requirements.txt delete mode 100755 library_generation/setup.py delete mode 100644 library_generation/templates/gapic-libraries-bom.xml.j2 delete mode 100644 library_generation/templates/owlbot.py.j2 delete mode 100644 library_generation/templates/owlbot.yaml.monorepo.j2 delete mode 100644 library_generation/templates/root-pom.xml.j2 delete mode 100644 library_generation/test/__init__.py delete mode 100644 library_generation/test/cli/__init__.py delete mode 100644 library_generation/test/cli/entry_point_unit_tests.py delete mode 100644 library_generation/test/cli/generate_release_note_unit_tests.py delete mode 100644 library_generation/test/compare_poms.py delete mode 100644 library_generation/test/generate_library_unit_tests.py delete mode 100755 library_generation/test/generate_library_unit_tests.sh delete mode 100644 library_generation/test/generate_pr_description_unit_tests.py delete mode 100644 library_generation/test/generate_repo_unit_tests.py delete mode 100644 library_generation/test/integration_tests.py delete mode 100644 library_generation/test/model/__init__.py delete mode 100644 library_generation/test/model/config_change_unit_tests.py delete mode 100644 library_generation/test/model/gapic_config_unit_tests.py delete mode 100644 library_generation/test/model/gapic_inputs_unit_tests.py delete mode 100644 library_generation/test/model/generation_config_unit_tests.py delete mode 100644 library_generation/test/model/library_config_unit_tests.py delete mode 100644 library_generation/test/model/repo_config_unit_tests.py delete mode 100644 library_generation/test/owlbot/__init__.py delete mode 100644 library_generation/test/owlbot/fix_poms_unit_tests.py delete mode 100644 library_generation/test/owlbot/java_unit_tests.py delete mode 100644 library_generation/test/owlbot/util.py delete mode 100644 library_generation/test/resources/gapic_options/QueryServiceGrpc_copy.java delete mode 100644 library_generation/test/resources/gapic_options/example.yaml delete mode 100644 library_generation/test/resources/gapic_options/example_gapic.legacy.yaml delete mode 100644 library_generation/test/resources/gapic_options/example_gapic.yaml delete mode 100644 library_generation/test/resources/gapic_options/example_gapic_legacy.yaml delete mode 100644 library_generation/test/resources/gapic_options/example_grpc_service_config.json delete mode 100644 library_generation/test/resources/goldens/.OwlBot-hermetic-golden.yaml delete mode 100644 library_generation/test/resources/goldens/.repo-metadata-custom-transport-golden.json delete mode 100644 library_generation/test/resources/goldens/.repo-metadata-monorepo-golden.json delete mode 100644 library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json delete mode 100644 library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json delete mode 100644 library_generation/test/resources/goldens/owlbot-golden.py delete mode 100644 library_generation/test/resources/goldens/pr_description-golden.txt delete mode 100644 library_generation/test/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt delete mode 100644 library_generation/test/resources/goldens/repo_level_only_pr_description-golden.txt delete mode 100644 library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml delete mode 100644 library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml delete mode 100644 library_generation/test/resources/integration/google-cloud-java/pr-description-golden.txt delete mode 100644 library_generation/test/resources/integration/java-bigtable/generation_config.yaml delete mode 100644 library_generation/test/resources/integration/java-bigtable/pr-description-golden.txt delete mode 100644 library_generation/test/resources/integration/test_generator_coordinates delete mode 100644 library_generation/test/resources/misc/BUILD_comment_common_resources.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_comment_iam_policy.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_comment_locations.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_common_resources.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_gapic_yaml.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_grpc.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_grpc_rest.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_iam_locations.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_iam_policy.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_include_samples_empty.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_include_samples_false.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_include_samples_true.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_locations.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_no_additional_protos.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_no_gapic_yaml.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_no_service_config.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_no_service_yaml.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_proto_only.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_rest.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_rest_numeric_enums_empty.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_rest_numeric_enums_false.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_rest_numeric_enums_true.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_service_config.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_service_config_relative_target.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_service_yaml.bazel delete mode 100644 library_generation/test/resources/misc/BUILD_service_yaml_absolute_target.bazel delete mode 100644 library_generation/test/resources/misc/TESTWORKSPACE delete mode 100644 library_generation/test/resources/misc/versions.txt delete mode 100644 library_generation/test/resources/proto_path_library/proto-test-library/src/main/proto/google/cloud/test/v1/empty.proto delete mode 100644 library_generation/test/resources/proto_path_library_multiple_protos/proto-1/fake.proto delete mode 100644 library_generation/test/resources/proto_path_library_multiple_protos/proto-2/fake.proto delete mode 100644 library_generation/test/resources/test-config/config_without_api_description.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_api_shortname.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_gapics_key.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_gapics_value.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_googleapis.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_libraries.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_library_value.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_name_pretty.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_product_docs.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_proto_path.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_temp_excludes.yaml delete mode 100644 library_generation/test/resources/test-config/generation_config.yaml delete mode 100644 library_generation/test/resources/test-config/generation_config_library_modified.yaml delete mode 100644 library_generation/test/resources/test-config/generation_config_with_duplicate_library_name.yaml delete mode 100644 library_generation/test/resources/test-config/monorepo_baseline.yaml delete mode 100644 library_generation/test/resources/test-config/monorepo_current.yaml delete mode 100644 library_generation/test/resources/test-config/monorepo_with_common_protos.yaml delete mode 100644 library_generation/test/resources/test-config/monorepo_without_common_protos.yaml delete mode 100644 library_generation/test/resources/test-monorepo/.github/.OwlBot.lock.yaml delete mode 100644 library_generation/test/resources/test-monorepo/test-service/.repo-metadata.json delete mode 100644 library_generation/test/resources/test-owlbot/fixtures/java_templates/release-please-update/.github/release-please.yml delete mode 100644 library_generation/test/resources/test-owlbot/fixtures/java_templates/release-please-update/.repo-metadata.json delete mode 100644 library_generation/test/resources/test-owlbot/fixtures/java_templates/render-readme/.repo-metadata.json delete mode 100644 library_generation/test/resources/test-owlbot/fixtures/java_templates/standard/.repo-metadata.json delete mode 100644 library_generation/test/resources/test-owlbot/java-admanager/.repo-metadata.json delete mode 100644 library_generation/test/resources/test-owlbot/java-admanager/ad-manager-bom/pom-golden.xml delete mode 100644 library_generation/test/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml delete mode 100644 library_generation/test/resources/test-owlbot/java-admanager/pom-golden.xml delete mode 100644 library_generation/test/resources/test-owlbot/java-admanager/proto-ad-manager-v1/pom-golden.xml delete mode 100644 library_generation/test/resources/test-owlbot/java-admanager/versions.txt delete mode 100644 library_generation/test/resources/test-owlbot/testdata/FooGrpcGolden.java delete mode 100644 library_generation/test/resources/test-owlbot/testdata/FooProtoGolden.java delete mode 100644 library_generation/test/resources/test-owlbot/testdata/README-golden.md delete mode 100644 library_generation/test/resources/test-owlbot/testdata/SampleClass.java delete mode 100644 library_generation/test/resources/test-owlbot/testdata/SampleClassGolden.java delete mode 100644 library_generation/test/resources/test-owlbot/testdata/SampleCopyMethodGolden.java delete mode 100644 library_generation/test/resources/test-owlbot/testdata/SampleDeprecateClass.java delete mode 100644 library_generation/test/resources/test-owlbot/testdata/SampleDeprecateMethodGolden.java delete mode 100644 library_generation/test/resources/test-owlbot/testdata/src/foo/FooGrpc.java delete mode 100644 library_generation/test/resources/test-owlbot/testdata/src/foo/FooProto.java delete mode 100644 library_generation/test/resources/test_generate_release_note/empty_gen_config.yaml delete mode 100644 library_generation/test/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml delete mode 100644 library_generation/test/resources/test_monorepo_postprocessing/java-dns/pom.xml delete mode 100644 library_generation/test/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml delete mode 100644 library_generation/test/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml delete mode 100644 library_generation/test/resources/test_monorepo_postprocessing/pom-golden.xml delete mode 100644 library_generation/test/resources/test_monorepo_postprocessing/versions.txt delete mode 100644 library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt delete mode 100644 library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt delete mode 100644 library_generation/test/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt delete mode 100644 library_generation/test/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt delete mode 100644 library_generation/test/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt delete mode 100755 library_generation/test/test_utilities.sh delete mode 100644 library_generation/test/test_utils.py delete mode 100644 library_generation/test/utilities_unit_tests.py delete mode 100644 library_generation/test/utils/__init__.py delete mode 100644 library_generation/test/utils/commit_message_formatter_unit_tests.py delete mode 100644 library_generation/test/utils/generation_config_comparator_unit_tests.py delete mode 100644 library_generation/test/utils/monorepo_postprocessor_unit_tests.py delete mode 100644 library_generation/test/utils/pom_generator_unit_tests.py delete mode 100644 library_generation/test/utils/proto_path_utils_unit_tests.py delete mode 100644 library_generation/utils/__init__.py delete mode 100644 library_generation/utils/commit_message_formatter.py delete mode 100644 library_generation/utils/file_render.py delete mode 100644 library_generation/utils/generation_config_comparator.py delete mode 100644 library_generation/utils/monorepo_postprocessor.py delete mode 100644 library_generation/utils/pom_generator.py delete mode 100644 library_generation/utils/proto_path_utils.py delete mode 100755 library_generation/utils/utilities.py delete mode 100755 library_generation/utils/utilities.sh diff --git a/hermetic_build/library_generation/DEVELOPMENT.md b/hermetic_build/library_generation/DEVELOPMENT.md new file mode 100644 index 0000000000..f32b544e1e --- /dev/null +++ b/hermetic_build/library_generation/DEVELOPMENT.md @@ -0,0 +1,207 @@ +> [!IMPORTANT] +> All examples assume you are inside the `library_generation` folder. + + +# Linting + +When contributing, ensure your changes to python code have a valid format. + +``` +python -m pip install black +black . +``` + +# Running the integration tests + +The integration tests build the docker image declared in +`.cloudbuild/library_generation/library_generation.Dockerfile`, pull GAPIC +repositories, generate the libraries and compares the results with the source +code declared in a "golden branch" of the repo. + +It requires docker and python 3.x to be installed. + +``` +python -m pip install . +python -m pip install -r requirements.txt +python -m unittest test/integration_tests.py +``` + +# Running the unit tests + +The unit tests of the hermetic build scripts are contained in several scripts, +corresponding to a specific component. +Every unit test script ends with `unit_tests.py`. +To avoid them specifying them individually, we can use the following command: + +```bash +python -m unittest discover -s test/ -p "*unit_tests.py" +``` + +> [!NOTE] +> The output of this command may look erratic during the first 30 seconds. +> This is normal. After the tests are done, an "OK" message should be shown. + +# Running the scripts in your local environment + +Although the scripts are designed to be run in a Docker container, you can also +run them directly. +This section explains how to run the entrypoint script +(`library_generation/cli/entry_point.py`). + +## Assumptions made by the scripts +### The Hermetic Build's well-known folder +Located in `${HOME}/.library_generation`, this folder is assumed by the scripts +to contain certain tools. + +Developers must make sure this folder is properly configured before running the +scripts locally. +Note that this relies on the `HOME` en var which is always defined as per +[POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html). + +#### Put the gapic-generator-java jar in its well-known location + +Run `cd sdk-platform-java && mvn install -DskipTests -Dclirr.skip +-Dcheckstyle.skip`. +This will generate a jar located in +`~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar` + +Then `mv` the jar into the well-known location of the jar. +The generation scripts will assume the jar is there. + +```shell +mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar" +``` + +#### Put the java formatter jar in its well-known location + +Download google-java-format-{version}-all-deps.jar from [Maven Central](https://central.sonatype.com/artifact/com.google.googlejavaformat/google-java-format) +or [GitHub releases](https://github.com/google/google-java-format/releases). +Then `mv` the jar into the well-known location of the jar. +The generation scripts will assume the jar is there. + +```shell +mv /path/to/jar "${HOME}/.library_generation/google-java-format.jar" +``` + +## Installing prerequisites + +In order to run the generation scripts directly, there are a few tools we +need to install beforehand. + +### Install the owl-bot CLI + +Requires node.js to be installed. +Check this [installation guide](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script) +for NVM, Node.js's version manager. + +After you install it, you can install the owl-bot CLI with the following +commands: +```bash +git clone https://github.com/googleapis/repo-automation-bots +cd repo-automation-bots/packages/owl-bot +npm i && npm run compile && npm link +owl-bot copy-code --version +``` + +The key step is `npm link`, which will make the command available in you current +shell session. + + +## Running the script +The entrypoint script (`library_generation/cli/entry_point.py`) allows you to +generate a GAPIC repository with a given api definition (proto, service yaml). + +### Download the api definition +For example, googleapis +``` +git clone https://github.com/googleapis/googleapis +export api_definitions_path="$(pwd)/googleapis" +``` + +### Download the repo +For example, google-cloud-java +``` +git clone https://github.com/googleapis/google-cloud-java +export path_to_repo="$(pwd)/google-cloud-java" +``` + +### Install the scripts +``` +python -m pip install . +``` + +### Run the script +``` +python cli/entry_point.py generate \ + --repository-path="${path_to_repo}" \ + --api-definitions-path="${api_definitions_path}" +``` + + +# Running the scripts using the docker container image +This is convenient in order to avoid installing the dependencies manually. + +> [!IMPORTANT] +> From now, the examples assume you are in the root of your sdk-platform-java +> folder. + +## Build the docker image +```bash +docker build --file .cloudbuild/library_generation/library_generation.Dockerfile --iidfile image-id . +``` + +This will create an `image-id` file at the root of the repo with the hash ID of +the image. + +## Run the docker image +The docker image will perform changes on its internal `/workspace` folder, +to which you need to map a folder on your host machine (i.e. map your downloaded +repo to this folder). + +To run the docker container on the google-cloud-java repo, you must run: +```bash +docker run \ + -u "$(id -u)":"$(id -g)" \ + -v /path/to/google-cloud-java:/workspace \ + -v /path/to/api-definition:/workspace/apis \ + $(cat image-id) \ + --api-definitions-path=/workspace/apis +``` + + * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating + yourself. This avoids folder ownership changes since it runs as root by + default. + * `-v /path/to/google-cloud-java:/workspace` maps the host machine's + google-cloud-java folder to the /workspace folder. + The image is configured to perform changes in this directory. + * `-v /path/to/api-definition:/workspace/apis` maps the host machine's + api-definition folder to /workspace/apis folder. + * `$(cat image-id)` obtains the image ID created in the build step. + * `--api-definitions-path=/workspace/apis` set the API definition path to + `/workspace/apis`. + +## Debug the created containers +If you are working on changing the way the containers are created, you may want +to inspect the containers to check the setup. +It would be convenient in such case to have a text editor/viewer available. +You can achieve this by modifying the Dockerfile as follows: + +```docker +# install OS tools +RUN apt-get update && apt-get install -y \ + unzip openjdk-17-jdk rsync maven jq less vim \ + && apt-get clean +``` + +We add `less` and `vim` as text tools for further inspection. + +You can also run a shell in a new container by running: + +```bash +docker run \ + --rm -it \ + -u $(id -u):$(id -g) \ + -v /path/to/google-cloud-java:/workspace \ + --entrypoint="bash" \ + $(cat image-id) +``` diff --git a/hermetic_build/library_generation/README.md b/hermetic_build/library_generation/README.md new file mode 100644 index 0000000000..518d588f9a --- /dev/null +++ b/hermetic_build/library_generation/README.md @@ -0,0 +1,310 @@ +# Generate a repository containing GAPIC Client Libraries + +The script, `entry_point.py`, allows you to generate a repository containing +GAPIC client libraries (a monorepo, for example, google-cloud-java) from a +configuration file. + +## Environment + +- OS: Linux +- Java runtime environment (8 or above) +- Python (3.11.6 or above) +- Docker +- Git + +## Prerequisite + +In order to generate a version for each library, a versions.txt has to exist +in `repository_path`. +Please refer to [Repository path](#repository-path--repositorypath---optional) for more information. + +## Parameters to generate a repository using `entry_point.py` + +### Baseline generation configuration yaml (`baseline_generation_config`) + +An absolute or relative path to a generation_config.yaml. +This config file is used for computing changed libraries, not library +generation. + +### Current generation configuration yaml (`current_generation_config`) + +An absolute or relative path to a configuration file containing parameters to +generate the repository. +Please refer [Configuration to generate a repository](#configuration-to-generate-a-repository) +for more information. + +### Repository path (`repository_path`), optional + +The path to where the generated repository goes. + +The default value is the current working directory when running the script. +For example, `cd google-cloud-java && python entry_point.py ...` without +specifying the `--repository_path` option will modify the `google-cloud-java` +repository the user `cd`'d into. + +Note that versions.txt has to exist in `repository_path` in order to generate +right version for each library. +Please refer [here](go/java-client-releasing#versionstxt-manifest) for more info +of versions.txt. + +### Api definitions path (`api_definitions_path`), optional + +The path to where the api definition (proto, service yaml) resides. + +The default value is the current working directory when running the script. + +Note that you need not only the protos defined the service, but also the transitive +dependencies of those protos. +Any missing dependencies will cause `File not found` error. + +For example, if your service is defined in `example_service.proto` and it imports +`google/api/annotations.proto`, you need the `annotations.proto` resides in a +folder that has the exact structure of the import statement (`google/api` in this +case), and set `api_definitions_path` to the path contains the root folder (`google` +in this case). + +## Output of `entry_point.py` + +### GAPIC libraries + +For each module (e.g. `google-cloud-java/java-asset`), the following files/folders +will be created/modified: + +| Name | Notes | +|:------------------------------------|:-------------------------------------------------------------------------| +| google-*/ | Source code generated by gapic-generator-java | +| google-*/pom.xml | Only be generated if it does not exist | +| grpc-*/ | Source code generated by grpc generator, one per each version | +| grpc-*/pom.xml | Only be generated if it does not exist | +| proto-*/ | Source code generated by Protobuf default compiler, one per each version | +| proto-*/pom.xml | Only be generated if it does not exist | +| samples/snippets/generated/ | Only be generated if `include_samples` is set to true | +| google-*-bom/pom.xml | Library BOM, only be generated if it does not exist | +| pom.xml | Library parent BOM, only be generated if it does not exist | +| .repo-metadata.json | Always generated from inputs | +| .OwlBot-hermetic.yaml | Only be generated from a template if it does not exist | +| owlbot.py | Only be generated from a template if it does not exist | +| README.md | Always generated from inputs | +| gapic-libraries-bom/pom.xml | Always generated from inputs | +| pom.xml (repo root dir) | Always generated from inputs | +| versions.txt | New entries will be added if they don’t exist | + +## Configuration to generate a repository + +There are three levels of parameters in the configuration: repository level, +library level and GAPIC level. + +### Repository level parameters + +The repository level parameters define the version of API definition (proto) +and tools. +They are shared by library level parameters. + +| Name | Required | Notes | +|:------------------------|:--------:|:---------------------------------------------| +| gapic_generator_version | No | set through env variable if not specified | +| protoc_version | No | inferred from the generator if not specified | +| grpc_version | No | inferred from the generator if not specified | +| googleapis_commitish | Yes | | +| libraries_bom_version | No | empty string if not specified | + +### Library level parameters + +The library level parameters define how to generate a (multi-versions) GAPIC +library. +They are shared by all GAPICs of a library. + +| Name | Required | Notes | +|:----------------------|:--------:|:------------------------------------------------------------------------------------------------------------------------------------------| +| api_shortname | Yes | | +| api_description | Yes | | +| name_pretty | Yes | | +| product_docs | Yes | | +| library_type | No | `GAPIC_AUTO` if not specified | +| release_level | No | `preview` if not specified | +| api_id | No | `{api_shortname}.googleapis.com` if not specified | +| api_reference | No | | +| codeowner_team | No | | +| client_documentation | No | | +| distribution_name | No | `{group_id}:google-{cloud_prefix}{library_name}` if not specified | +| excluded_poms | No | | +| excluded_dependencies | No | | +| googleapis_commitish | No | use repository level `googleapis_commitish` if not specified. | +| group_id | No | `com.google.cloud` if not specified | +| issue_tracker | No | | +| library_name | No | `api_shortname` is not specified. This value should be unique among all libraries. | +| rest_documentation | No | | +| rpc_documentation | No | | +| cloud_api | No | `true` if not specified | +| requires-billing | No | `true` if not specified | +| transport | No | must be one of `grpc`, `rest` or `both`. This value would only be used for generating .repo-metadata.json and relevant sections in README | + + +Note that `cloud_prefix` is `cloud-` if `cloud_api` is `true`; empty otherwise. + +### GAPIC level parameters + +The GAPIC level parameters define how to generate a GAPIC library. + +| Name | Required | Notes | +|:-----------|:--------:|:------------------------------------------| +| proto_path | Yes | versioned proto_path starts with `google` | + +### An example of generation configuration + +```yaml +gapic_generator_version: 2.34.0 +protoc_version: 25.2 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." + release_level: "stable" + library_name: "apigee-connect" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 + + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: "stable" + issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" + api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 +``` + +# Local Environment Setup before running `entry_point.py` + +1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. +The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case). +2. Assuming the virtual environment is installed under `sdk-platform-java`. +Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation` + + ```bash + python -m pip install --require-hashes -r library_generation/requirements.txt + ``` + +3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere + ```bash + python -m pip install library_generation/ + ``` + +4. Download api definition to a local directory + +## An example to generate a repository using `entry_point.py` + +```bash +python library_generation/entry_point.py generate \ +--baseline-generation-config-path=/path/to/baseline_config_file \ +--current-generation-config-path=/path/to/current_config_file \ +--repository-path=path/to/repository \ +--api-definitions-path=path/to/api_definition +``` +If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration) +shown above, the repository structure is: +``` +$repository_path +|_gapic-libraries-bom +| |_pom.xml +|_java-apigee-connect +| |_google-cloud-apigee-connect +| | |_src +| | |_pom.xml +| |_google-cloud-apigee-connect-bom +| | |_pom.xml +| |_grpc-google-cloud-apigee-connect-v1 +| | |_src +| | |_pom.xml +| |_proto-google-cloud-apigee-connect-v1 +| | |_src +| | |_pom.xml +| |_samples +| | |_snippets +| | | |_generated +| |_.OwlBot-hermetic.yaml +| |_.repo-metadata.json +| |_owlbot.py +| |_pom.xml +| |_README.md +|_java-asset +| |_google-cloud-asset +| | |_src +| | |_pom.xml +| |_google-cloud-asset-bom +| | |_pom.xml +| |_grpc-google-cloud-asset-v1 +| | |_src +| | |_pom.xml +| |_grpc-google-cloud-asset-v1p1beta1 +| | |_src +| | |_pom.xml +| |_grpc-google-cloud-asset-v1p2beta1 +| | |_src +| | |_pom.xml +| |_grpc-google-cloud-asset-v1p5beta1 +| | |_src +| | |_pom.xml +| |_grpc-google-cloud-asset-v1p7beta1 +| | |_src +| | |_pom.xml +| |_proto-google-cloud-asset-v1 +| | |_src +| | |_pom.xml +| |_proto-google-cloud-asset-v1p1beta1 +| | |_src +| | |_pom.xml +| |_proto-google-cloud-asset-v1p2beta1 +| | |_src +| | |_pom.xml +| |_proto-google-cloud-asset-v1p5beta1 +| | |_src +| | |_pom.xml +| |_proto-google-cloud-asset-v1p7beta1 +| | |_src +| | |_pom.xml +| |_samples +| | |_snippets +| | | |_generated +| |_.OwlBot-hermetic.yaml +| |_.repo-metadata.json +| |_owlbot.py +| |_pom.xml +| |_README.md +|_pom.xml +|_versions.txt +``` + +# Owlbot Java Postprocessor + +We have transferred the +[implementation](https://github.com/googleapis/synthtool/tree/59fe44fde9866a26e7ee4e4450fd79f67f8cf599/docker/owlbot/java) +of Java Owlbot Postprocessor into `sdk-platform-java/library_generation`. The +implementation in synthtool is still valid and used by other services, so we +have two versions during a transition period. + +## Reflecting changes in synthtool/docker/owlbot/java into this repository +The transfer was not a verbatim copy, it rather had modifications: + * `format-source.sh` was replaced by a call to `mvn fmt:format` + * `entrypoint.sh` was modified to have input arguments and slightly modified + the way the helper scripts are called + * Other helper scripts were modified to have input arguments. + * `fix_poms.py` modified the way the monorepo is detected + +All these modifications imply that whenever we want to reflect a change from the +original owlbot in synthtool we may be better off modifying the affected source +files one by one. The mapping is from +[`synthtool/docker/owlbot/java`](https://github.com/googleapis/synthtool/tree/59fe44fde9866a26e7ee4e4450fd79f67f8cf599/docker/owlbot/java) +to +[`sdk-platform-java/library_generation/owlbot`](https://github.com/googleapis/sdk-platform-java/tree/move-java-owlbot/library_generation/owlbot) diff --git a/library_generation/__init__.py b/library_generation/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py deleted file mode 100644 index a41d25e500..0000000000 --- a/library_generation/cli/entry_point.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import sys -from typing import Optional -import click as click -from library_generation.generate_repo import generate_from_yaml -from library_generation.model.config_change import ConfigChange -from library_generation.model.generation_config import from_yaml -from library_generation.utils.generation_config_comparator import compare_config - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--baseline-generation-config-path", - required=False, - default=None, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml. - This config file is used for commit history generation, not library - generation. - """, -) -@click.option( - "--current-generation-config-path", - required=False, - default=None, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml that contains the - metadata about library generation. - """, -) -@click.option( - "--library-names", - type=str, - default=None, - show_default=True, - help=""" - A list of library names that will be generated, separated by comma. - The library name of a library is the value of library_name or api_shortname, - if library_name is not specified, in the generation configuration. - """, -) -@click.option( - "--repository-path", - type=str, - default=".", - show_default=True, - help=""" - The repository path to which the generated files - will be sent. - If not specified, the repository will be generated to the current working - directory. - """, -) -@click.option( - "--api-definitions-path", - type=str, - default=".", - show_default=True, - help=""" - The path to which the api definition (proto and service yaml) and its - dependencies resides. - If not specified, the path is the current working directory. - """, -) -def generate( - baseline_generation_config_path: str, - current_generation_config_path: str, - library_names: Optional[str], - repository_path: str, - api_definitions_path: str, -): - """ - Compare baseline generation config and current generation config and - generate changed libraries based on current generation config. - - If baseline generation config is not specified but current generation - config is specified, generate all libraries if `library_names` is not - specified, based on current generation config. - - If current generation config is not specified but baseline generation - config is specified, raise FileNotFoundError because current generation - config should be the source of truth of library generation. - - If both baseline generation config and current generation config are not - specified, generate all libraries based on the default generation config, - which is generation_config.yaml in the current working directory. - - If `library_names` is specified, only libraries whose name can be found in - the current generation config or default generation config, if current - generation config is not specified, will be generated. Changed libraries - will be ignored even if baseline and current generation config are - specified. - - Raise FileNotFoundError if the default config does not exist. - """ - __generate_repo_and_pr_description_impl( - baseline_generation_config_path=baseline_generation_config_path, - current_generation_config_path=current_generation_config_path, - library_names=library_names, - repository_path=repository_path, - api_definitions_path=api_definitions_path, - ) - - -def __generate_repo_and_pr_description_impl( - baseline_generation_config_path: str, - current_generation_config_path: str, - library_names: Optional[str], - repository_path: str, - api_definitions_path: str, -): - """ - Implementation method for generate(). - The decoupling of generate and __generate_repo_and_pr_description_impl is - meant to allow testing of this implementation function. - """ - - default_generation_config_path = f"{os.getcwd()}/generation_config.yaml" - - if ( - baseline_generation_config_path is None - and current_generation_config_path is None - ): - if not os.path.isfile(default_generation_config_path): - raise FileNotFoundError( - f"{default_generation_config_path} does not exist. " - "A valid generation config has to be passed in as " - "current_generation_config or exist in the current working " - "directory." - ) - current_generation_config_path = default_generation_config_path - elif current_generation_config_path is None: - raise FileNotFoundError( - "current_generation_config is not specified when " - "baseline_generation_config is specified. " - "current_generation_config should be the source of truth of " - "library generation." - ) - - current_generation_config_path = os.path.abspath(current_generation_config_path) - repository_path = os.path.abspath(repository_path) - api_definitions_path = os.path.abspath(api_definitions_path) - include_library_names = _parse_library_name_from(library_names) - - if not baseline_generation_config_path: - # Execute selective generation based on current_generation_config if - # baseline_generation_config is not specified. - generate_from_yaml( - config=from_yaml(current_generation_config_path), - repository_path=repository_path, - api_definitions_path=api_definitions_path, - target_library_names=include_library_names, - ) - return - - # Compare two generation configs to get changed libraries. - baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) - config_change = compare_config( - baseline_config=from_yaml(baseline_generation_config_path), - current_config=from_yaml(current_generation_config_path), - ) - # Pass None if we want to fully generate the repository. - changed_library_names = ( - config_change.get_changed_libraries() - if not _needs_full_repo_generation(config_change=config_change) - else None - ) - # Include library names takes preference if specified. - target_library_names = ( - include_library_names - if include_library_names is not None - else changed_library_names - ) - generate_from_yaml( - config=config_change.current_config, - repository_path=repository_path, - api_definitions_path=api_definitions_path, - target_library_names=target_library_names, - ) - - -def _needs_full_repo_generation(config_change: ConfigChange) -> bool: - """ - Whether you should need a full repo generation, i.e., generate all - libraries in the generation configuration. - """ - current_config = config_change.current_config - return not current_config.is_monorepo() or current_config.contains_common_protos() - - -def _parse_library_name_from(includes: str) -> Optional[list[str]]: - if includes is None: - return None - return [library_name.strip() for library_name in includes.split(",")] - - -@main.command() -@click.option( - "--generation-config-path", - required=False, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml. - Default to generation_config.yaml in the current working directory. - """, -) -def validate_generation_config(generation_config_path: str) -> None: - """ - Validate the given generation configuration. - """ - if generation_config_path is None: - generation_config_path = "generation_config.yaml" - try: - from_yaml(os.path.abspath(generation_config_path)) - print(f"{generation_config_path} is validated without any errors.") - except ValueError as err: - print(err) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/library_generation/cli/generate_monorepo_gapic_bom.py b/library_generation/cli/generate_monorepo_gapic_bom.py deleted file mode 100644 index b26a02c17f..0000000000 --- a/library_generation/cli/generate_monorepo_gapic_bom.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import click as click - -from library_generation.utils.pom_generator import generate_gapic_bom - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--repository-path", - required=True, - type=str, - help=""" - Path to which the generated pom.xml goes. - """, -) -@click.option( - "--versions-file", - required=True, - type=str, - help=""" - The file containing version of libraries. - Throw FileNotFoundError if the file doesn't exist. - """, -) -def generate(repository_path: str, versions_file: str) -> None: - generate_gapic_bom(repository_path=repository_path, versions_file=versions_file) - - -if __name__ == "__main__": - main() diff --git a/library_generation/cli/generate_monorepo_root_pom.py b/library_generation/cli/generate_monorepo_root_pom.py deleted file mode 100644 index 8f129b63f0..0000000000 --- a/library_generation/cli/generate_monorepo_root_pom.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import click as click -from library_generation.utils.pom_generator import generate_root_pom - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--repository-path", - required=True, - type=str, - help=""" - Path to which the generated pom.xml goes. - """, -) -def generate(repository_path: str) -> None: - generate_root_pom(repository_path=repository_path) - - -if __name__ == "__main__": - main() diff --git a/library_generation/cli/generate_release_note.py b/library_generation/cli/generate_release_note.py deleted file mode 100644 index 097496e9a4..0000000000 --- a/library_generation/cli/generate_release_note.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from typing import Optional -import click as click -from library_generation.generate_pr_description import generate_pr_descriptions -from library_generation.model.generation_config import from_yaml -from library_generation.utils.generation_config_comparator import compare_config - - -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--baseline-generation-config-path", - required=False, - default=None, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml. - This config file is used for commit history generation, not library - generation. - """, -) -@click.option( - "--current-generation-config-path", - required=False, - default=None, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml that contains the - metadata about library generation. - """, -) -@click.option( - "--repository-path", - type=str, - default=".", - show_default=True, - help=""" - The repository path to which the generated files will be sent. - If not specified, the repository will be generated to the current working - directory. - """, -) -def generate( - baseline_generation_config_path: Optional[str], - current_generation_config_path: Optional[str], - repository_path: str, -) -> None: - if ( - baseline_generation_config_path is None - or current_generation_config_path is None - ): - print( - "One of the generation configs is not specified, do not generate " - "the description." - ) - return - baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) - current_generation_config_path = os.path.abspath(current_generation_config_path) - if not ( - os.path.isfile(baseline_generation_config_path) - and os.path.isfile(current_generation_config_path) - ): - print( - "One of the generation configs does not exist, do not generate " - "the description." - ) - return - config_change = compare_config( - baseline_config=from_yaml(baseline_generation_config_path), - current_config=from_yaml(current_generation_config_path), - ) - generate_pr_descriptions( - config_change=config_change, - description_path=os.path.abspath(repository_path), - ) - - -if __name__ == "__main__": - main() diff --git a/library_generation/dockerignore b/library_generation/dockerignore deleted file mode 100644 index 7e978caea9..0000000000 --- a/library_generation/dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -README.md -**/__pycache__/ -**/*.egg-info/ -**/output/ -**/build/ -**/google-cloud-java/ diff --git a/library_generation/gapic-generator-java-wrapper b/library_generation/gapic-generator-java-wrapper deleted file mode 100755 index 3f1e7bd2e7..0000000000 --- a/library_generation/gapic-generator-java-wrapper +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -e -wrapper_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") -source "${wrapper_dir}/utils/utilities.sh" - -# Wrap gapic-generator-java.jar because protoc requires the plugin to be executable. -exec java -classpath "$(get_gapic_generator_location)" com.google.api.generator.Main diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py deleted file mode 100755 index 51d1ef21f3..0000000000 --- a/library_generation/generate_composed_library.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This script allows generation of libraries that are composed of more than one -service version. It is achieved by calling `generate_library.sh` without -postprocessing for all service versions and then calling -postprocess_library.sh at the end, once all libraries are ready. - -Prerequisites -- Needs a folder named `output` in current working directory. This folder -is automatically detected by `generate_library.sh` and this script ensures it -contains the necessary folders and files, specifically: - - A "google" folder found in the googleapis/googleapis repository - - A "grafeas" folder found in the googleapis/googleapis repository -Note: googleapis repo is found in https://github.com/googleapis/googleapis. -""" -import os -from pathlib import Path -from typing import List -import library_generation.utils.utilities as util -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.gapic_inputs import GapicInputs -from library_generation.model.library_config import LibraryConfig -from library_generation.model.gapic_inputs import parse as parse_build_file -from library_generation.model.repo_config import RepoConfig - -script_dir = os.path.dirname(os.path.realpath(__file__)) - - -def generate_composed_library( - config: GenerationConfig, - library_path: str, - library: LibraryConfig, - repo_config: RepoConfig, -) -> None: - """ - Generate libraries composed of more than one service or service version - - :param config: a GenerationConfig object representing a parsed configuration - yaml - :param library_path: the path to which the generated file goes - :param library: a LibraryConfig object contained inside config, passed here - for convenience and to prevent all libraries to be processed - :param repo_config: - :return None - """ - output_folder = repo_config.output_folder - base_arguments = __construct_tooling_arg(config=config) - owlbot_cli_source_folder = util.sh_util("mktemp -d") - os.makedirs(f"{library_path}", exist_ok=True) - for gapic in library.get_sorted_gapic_configs(): - build_file_folder = Path(f"{output_folder}/{gapic.proto_path}").resolve() - print(f"build_file_folder: {build_file_folder}") - gapic_inputs = parse_build_file(build_file_folder, gapic.proto_path) - # generate postprocessing prerequisite files (.repo-metadata.json, .OwlBot-hermetic.yaml, - # owlbot.py) here because transport is parsed from BUILD.bazel, - # which lives in a versioned proto_path. The value of transport will be - # overridden by the config object if specified. Note that this override - # does not affect library generation but instead used only for - # generating postprocessing files such as README. - util.generate_postprocessing_prerequisite_files( - config=config, - library=library, - proto_path=util.remove_version_from(gapic.proto_path), - library_path=library_path, - transport=library.get_transport(gapic_inputs), - ) - temp_destination_path = f"java-{gapic.proto_path.replace('/','-')}" - effective_arguments = __construct_effective_arg( - base_arguments=base_arguments, - gapic=gapic, - gapic_inputs=gapic_inputs, - temp_destination_path=temp_destination_path, - ) - print("arguments: ") - print(effective_arguments) - print(f"Generating library from {gapic.proto_path} to {library_path}") - util.run_process_and_print_output( - ["bash", f"{script_dir}/generate_library.sh", *effective_arguments], - "Library generation", - ) - - util.sh_util( - f'build_owlbot_cli_source_folder "{library_path}"' - + f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"' - + f' "{gapic.proto_path}"', - cwd=output_folder, - ) - - library_version = repo_config.get_library_version( - artifact_id=library.get_artifact_id() - ) - # call postprocess library - util.run_process_and_print_output( - [ - f"{script_dir}/postprocess_library.sh", - f"{library_path}", - "", - repo_config.versions_file, - owlbot_cli_source_folder, - str(config.is_monorepo()).lower(), - config.libraries_bom_version, - library_version, - ], - "Library postprocessing", - ) - - -def __construct_tooling_arg(config: GenerationConfig) -> List[str]: - """ - Construct arguments of tooling versions used in generate_library.sh - :param config: the generation config - :return: arguments containing tooling versions - """ - arguments = [] - arguments += util.create_argument("grpc_version", config) - arguments += util.create_argument("protoc_version", config) - - return arguments - - -def __construct_effective_arg( - base_arguments: List[str], - gapic: GapicConfig, - gapic_inputs: GapicInputs, - temp_destination_path: str, -) -> List[str]: - """ - Construct arguments consist attributes of a GAPIC library which used in - generate_library.sh - :param base_arguments: arguments consist of tooling versions - :param gapic: an object of GapicConfig - :param gapic_inputs: an object of GapicInput - :param temp_destination_path: the path to which the generated library goes - :return: arguments containing attributes to generate a GAPIC library - """ - arguments = list(base_arguments) - arguments += util.create_argument("proto_path", gapic) - arguments += [ - "--proto_only", - gapic_inputs.proto_only, - "--gapic_additional_protos", - gapic_inputs.additional_protos, - "--transport", - gapic_inputs.transport, - "--rest_numeric_enums", - gapic_inputs.rest_numeric_enum, - "--gapic_yaml", - gapic_inputs.gapic_yaml, - "--service_config", - gapic_inputs.service_config, - "--service_yaml", - gapic_inputs.service_yaml, - "--include_samples", - gapic_inputs.include_samples, - ] - arguments += ["--destination_path", temp_destination_path] - - return arguments diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh deleted file mode 100755 index b8d22aca54..0000000000 --- a/library_generation/generate_library.sh +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -# parse input parameters -while [[ $# -gt 0 ]]; do -key="$1" -case $key in - -p|--proto_path) - proto_path="$2" - shift - ;; - -d|--destination_path) - destination_path="$2" - shift - ;; - --protoc_version) - protoc_version="$2" - shift - ;; - --grpc_version) - grpc_version="$2" - shift - ;; - --proto_only) - proto_only="$2" - shift - ;; - --gapic_additional_protos) - gapic_additional_protos="$2" - shift - ;; - --transport) - transport="$2" - shift - ;; - --rest_numeric_enums) - rest_numeric_enums="$2" - shift - ;; - --gapic_yaml) - gapic_yaml="$2" - shift - ;; - --service_config) - service_config="$2" - shift - ;; - --service_yaml) - service_yaml="$2" - shift - ;; - --include_samples) - include_samples="$2" - shift - ;; - --os_architecture) - os_architecture="$2" - shift - ;; - *) - echo "Invalid option: [$1]" - exit 1 - ;; -esac -shift # past argument or value -done - -script_dir=$(dirname "$(readlink -f "$0")") -# source utility functions -source "${script_dir}"/utils/utilities.sh -output_folder="$(get_output_folder)" - -if [ -z "${protoc_version}" ]; then - protoc_version=$(get_protoc_version) -fi - -if [ -z "${grpc_version}" ]; then - grpc_version=$(get_grpc_version) -fi - -if [ -z "${proto_only}" ]; then - proto_only="false" -fi - -if [ -z "${gapic_additional_protos}" ]; then - gapic_additional_protos="google/cloud/common_resources.proto" -fi - -if [ -z "${transport}" ]; then - transport="grpc" -fi - -if [ -z "${rest_numeric_enums}" ]; then - rest_numeric_enums="true" -fi - -if [ -z "${gapic_yaml}" ]; then - gapic_yaml="" -fi - -if [ -z "${service_config}" ]; then - service_config="" -fi - -if [ -z "${service_yaml}" ]; then - service_yaml="" -fi - -if [ -z "${include_samples}" ]; then - include_samples="true" -fi - -if [ -z "${os_architecture}" ]; then - os_architecture=$(detect_os_architecture) -fi - -temp_destination_path="${output_folder}/temp_preprocessed" -mkdir -p "${output_folder}/${destination_path}" -if [ -d "${temp_destination_path}" ]; then - # we don't want the preprocessed sources of a previous run - rm -rd "${temp_destination_path}" -fi -mkdir -p "${temp_destination_path}" -##################### Section 0 ##################### -# prepare tooling -##################################################### -# the order of services entries in gapic_metadata.json is relevant to the -# order of proto file, sort the proto files with respect to their bytes to -# get a fixed order. -folder_name=$(extract_folder_name "${destination_path}") -pushd "${output_folder}" -find_depth="" -case "${proto_path}" in - "google/api" | "google/cloud" | "google/rpc") - find_depth="-maxdepth 1" - ;; -esac -proto_files=$(find "${proto_path}" ${find_depth} -type f -name "*.proto" | LC_COLLATE=C sort) -# include or exclude certain protos in grpc plugin and gapic generator java. -case "${proto_path}" in - "google/cloud") - # this proto is excluded from //google/cloud:google-apps-script-type-java - removed_proto="google/cloud/common_resources.proto" - proto_files="${proto_files//${removed_proto}/}" - ;; - "google/cloud/aiplatform/v1beta1"*) - # this proto is excluded from //google/cloud/aiplatform/v1beta1/schema:schema_proto - removed_proto="google/cloud/aiplatform/v1beta1/schema/io_format.proto" - proto_files="${proto_files//${removed_proto}/}" - ;; - "google/cloud/filestore"*) - # this proto is included in //google/cloud/filestore/v1:google-cloud-filestore-v1-java - # and //google/cloud/filestore/v1beta1:google-cloud-filestore-v1-java - proto_files="${proto_files} google/cloud/common/operation_metadata.proto" - ;; - "google/cloud/oslogin"*) - # this proto is included in //google/cloud/oslogin/v1:google-cloud-oslogin-v1-java - # and //google/cloud/oslogin/v1beta1:google-cloud-oslogin-v1-java - proto_files="${proto_files} google/cloud/oslogin/common/common.proto" - ;; - "google/cloud/visionai/v1"*) - # this proto is excluded in //google/cloud/visionai/v1:google-cloud-visionai-v1-java - # we can remove this exclusion after cl/631529749 is submitted. - removed_proto="google/cloud/visionai/v1/prediction.proto" - proto_files="${proto_files//${removed_proto}/}" - ;; - "google/rpc") - # this proto is excluded from //google/rpc:google-rpc-java - removed_proto="google/rpc/http.proto" - proto_files="${proto_files//${removed_proto}/}" - ;; -esac -# download gapic-generator-java, protobuf and grpc plugin. -# the download_tools function will create the environment variables "protoc_path" -# and "grpc_path", to be used in the protoc calls below. -download_tools "${protoc_version}" "${grpc_version}" "${os_architecture}" -##################### Section 1 ##################### -# generate grpc-*/ -##################################################### -if [[ ! "${transport}" == "rest" ]]; then - # do not need to generate grpc-* if the transport is `rest`. - "${protoc_path}"/protoc "--plugin=protoc-gen-rpc-plugin=${grpc_path}" \ - "--rpc-plugin_out=:${temp_destination_path}/java_grpc.jar" \ - ${proto_files} # Do not quote because this variable should not be treated as one long string. - # unzip java_grpc.jar to grpc-*/src/main/java - unzip_src_files "grpc" "${temp_destination_path}" - # remove empty files in grpc-*/src/main/java - remove_empty_files "grpc" "${temp_destination_path}" - # remove grpc version in *ServiceGrpc.java file so the content is identical with bazel build. - remove_grpc_version "${temp_destination_path}" -fi -###################### Section 2 ##################### -## generate gapic-*/, part of proto-*/, samples/ -###################################################### -if [[ "${proto_only}" == "false" ]]; then - "$protoc_path"/protoc --experimental_allow_proto3_optional \ - "--plugin=protoc-gen-java_gapic=${script_dir}/gapic-generator-java-wrapper" \ - "--java_gapic_out=metadata:${temp_destination_path}/java_gapic_srcjar_raw.srcjar.zip" \ - "--java_gapic_opt=$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "${gapic_yaml}" "${service_config}" "${service_yaml}")" \ - ${proto_files} ${gapic_additional_protos} - - unzip -o -q "${temp_destination_path}/java_gapic_srcjar_raw.srcjar.zip" -d "${temp_destination_path}" - # Sync'\''d to the output file name in Writer.java. - unzip -o -q "${temp_destination_path}/temp-codegen.srcjar" -d "${temp_destination_path}/java_gapic_srcjar" - # Resource name source files. - proto_dir=${temp_destination_path}/java_gapic_srcjar/proto/src/main/java - if [ ! -d "${proto_dir}" ]; then - # Some APIs don't have resource name helpers, like BigQuery v2. - # Create an empty file so we can finish building. Gating the resource name rule definition - # on file existences go against Bazel's design patterns, so we'll simply delete all empty - # files during the final packaging process (see java_gapic_pkg.bzl) - mkdir -p "${proto_dir}" - touch "${proto_dir}"/PlaceholderFile.java - fi - # move java_gapic_srcjar/src/main to gapic-*/src. - mv_src_files "gapic" "main" "${temp_destination_path}" - # remove empty files in gapic-*/src/main/java - remove_empty_files "gapic" "${temp_destination_path}" - # move java_gapic_srcjar/src/test to gapic-*/src - mv_src_files "gapic" "test" "${temp_destination_path}" - if [ "${include_samples}" == "true" ]; then - # move java_gapic_srcjar/samples/snippets to samples/snippets - mv_src_files "samples" "main" "${temp_destination_path}" - fi -fi -##################### Section 3 ##################### -# generate proto-*/ -##################################################### -# exclude certain protos to java compiler. -case "${proto_path}" in - "google/cloud/aiplatform/v1beta1"*) - # these protos are excluded from //google/cloud/aiplatform/v1beta1:google-cloud-aiplatform-v1beta1-java - prefix="google/cloud/aiplatform/v1beta1/schema" - protos="${prefix}/annotation_payload.proto ${prefix}/annotation_spec_color.proto ${prefix}/data_item_payload.proto ${prefix}/dataset_metadata.proto ${prefix}/geometry.proto" - for removed_proto in ${protos}; do - proto_files="${proto_files//${removed_proto}/}" - done - ;; -esac -"$protoc_path"/protoc "--java_out=${temp_destination_path}/java_proto.jar" ${proto_files} -if [[ "${proto_only}" == "false" ]]; then - # move java_gapic_srcjar/proto/src/main/java (generated resource name helper class) - # to proto-*/src/main - mv_src_files "proto" "main" "${temp_destination_path}" -fi -# unzip java_proto.jar to proto-*/src/main/java -unzip_src_files "proto" "${temp_destination_path}" -# remove empty files in proto-*/src/main/java -remove_empty_files "proto" "${temp_destination_path}" -case "${proto_path}" in - "google/cloud/aiplatform/v1beta1"*) - prefix="google/cloud/aiplatform/v1beta1/schema" - protos="${prefix}/annotation_payload.proto ${prefix}/annotation_spec_color.proto ${prefix}/data_item_payload.proto ${prefix}/dataset_metadata.proto ${prefix}/geometry.proto" - for added_proto in ${protos}; do - proto_files="${proto_files} ${added_proto}" - done - ;; -esac -# copy proto files to proto-*/src/main/proto -for proto_src in ${proto_files}; do - if [[ "${proto_src}" == "google/cloud/common/operation_metadata.proto" ]]; then - continue - fi - mkdir -p "${temp_destination_path}/proto-${folder_name}/src/main/proto" - rsync -R "${proto_src}" "${temp_destination_path}/proto-${folder_name}/src/main/proto" -done -popd # output_folder -##################### Section 4 ##################### -# rm tar files -##################################################### -pushd "${temp_destination_path}" -rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar -popd # destination path - -cp -r ${temp_destination_path}/* "${output_folder}/${destination_path}" -rm -rdf "${temp_destination_path}" -exit 0 diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py deleted file mode 100755 index ac2f19003d..0000000000 --- a/library_generation/generate_pr_description.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import calendar -import os -import shutil -from typing import Dict -from git import Commit, Repo - -from library_generation.model.config_change import ConfigChange -from library_generation.utils.proto_path_utils import find_versioned_proto_path -from library_generation.utils.commit_message_formatter import ( - format_commit_message, - format_repo_level_change, - commit_link, -) -from library_generation.utils.commit_message_formatter import wrap_override_commit - -EMPTY_MESSAGE = "" - - -def generate_pr_descriptions( - config_change: ConfigChange, - description_path: str, - repo_url: str = "https://github.com/googleapis/googleapis.git", -) -> None: - """ - Generate pull request description from configuration comparison result. - - The pull request description contains repo-level changes, if applicable, - and googleapis commit from baseline config (exclusive) to current config - (inclusive). - - The pull request description will be generated into - description_path/pr_description.txt. - - No pr_description.txt will be generated if no changes in the configurations. - - :param config_change: a ConfigChange object, containing changes in baseline - and current generation configurations. - :param description_path: the path to which the pull request description - file goes. - :param repo_url: the GitHub repository from which retrieves the commit - history. - """ - repo_level_message = format_repo_level_change(config_change) - paths = config_change.current_config.get_proto_path_to_library_name() - description = get_repo_level_commit_messages( - repo_url=repo_url, - current_commit_sha=config_change.current_config.googleapis_commitish, - baseline_commit_sha=config_change.baseline_config.googleapis_commitish, - paths=paths, - is_monorepo=config_change.current_config.is_monorepo(), - repo_level_message=repo_level_message, - ) - - if description == EMPTY_MESSAGE: - print("Empty commit messages, skip creating pull request description.") - return - - description_file = f"{description_path}/pr_description.txt" - print(f"Writing pull request description to {description_file}") - with open(description_file, "w+") as f: - f.write(description) - - -def get_repo_level_commit_messages( - repo_url: str, - current_commit_sha: str, - baseline_commit_sha: str, - paths: Dict[str, str], - is_monorepo: bool, - repo_level_message: list[str], -) -> str: - """ - Combine commit messages of a repository from latest_commit to - baseline_commit. Only commits which change files in a pre-defined - file paths will be considered. - Note that baseline_commit should be an ancestor of or the same as - latest_commit. - - :param repo_url: the url of the repository. - :param current_commit_sha: the newest commit to be considered in - selecting commit message. - :param baseline_commit_sha: the oldest commit to be considered in - selecting commit message. This commit should be an ancestor of - :param paths: a mapping from file paths to library_name. - :param is_monorepo: whether to generate commit messages in a monorepo. - :param repo_level_message: commit messages regarding repo-level changes. - :return: commit messages. - :raise ValueError: if current_commit is older than or equal to - baseline_commit. - """ - tmp_dir = "/tmp/repo" - shutil.rmtree(tmp_dir, ignore_errors=True) - os.mkdir(tmp_dir) - repo = Repo.clone_from(repo_url, tmp_dir) - current_commit = repo.commit(current_commit_sha) - baseline_commit = repo.commit(baseline_commit_sha) - current_commit_time = __get_commit_timestamp(current_commit) - baseline_commit_time = __get_commit_timestamp(baseline_commit) - if current_commit_time < baseline_commit_time: - raise ValueError( - f"current_commit ({current_commit_sha[:7]}, committed on " - f"{current_commit_time}) should be newer than or equal to " - f"baseline_commit ({baseline_commit_sha[:7]}, committed on " - f"{baseline_commit_time})." - ) - qualified_commits = {} - commit = current_commit - while str(commit.hexsha) != baseline_commit_sha: - commit_and_name = __filter_qualified_commit(paths=paths, commit=commit) - if commit_and_name != (): - qualified_commits[commit_and_name[0]] = commit_and_name[1] - commit_parents = commit.parents - if len(commit_parents) == 0: - break - commit = commit_parents[0] - shutil.rmtree(tmp_dir, ignore_errors=True) - - return __combine_commit_messages( - current_commit=current_commit, - baseline_commit=baseline_commit, - commits=qualified_commits, - is_monorepo=is_monorepo, - repo_level_message=repo_level_message, - ) - - -def __filter_qualified_commit(paths: Dict[str, str], commit: Commit) -> (Commit, str): - """ - Returns a tuple of a commit and libray_name. - A qualified commit means at least one file, excluding BUILD.bazel, changes - in that commit is within the versioned proto_path in paths. - - :param paths: a mapping from versioned proto_path to library_name. - :param commit: a commit under consideration. - :return: a tuple of a commit and library_name if the commit is - qualified; otherwise an empty tuple. - """ - for file in commit.stats.files.keys(): - versioned_proto_path = find_versioned_proto_path(file) - if versioned_proto_path in paths and (not file.endswith("BUILD.bazel")): - return commit, paths[versioned_proto_path] - return () - - -def __combine_commit_messages( - current_commit: Commit, - baseline_commit: Commit, - commits: Dict[Commit, str], - is_monorepo: bool, - repo_level_message: list[str], -) -> str: - description = [] - if current_commit != baseline_commit: - description.append( - f"This pull request is generated with proto changes between " - f"{commit_link(baseline_commit)} (exclusive) " - f"and {commit_link(current_commit)} (inclusive).\n", - ) - commit_message = repo_level_message - commit_message.extend( - format_commit_message(commits=commits, is_monorepo=is_monorepo) - ) - if len(commit_message) == 0: - return EMPTY_MESSAGE - description.extend(wrap_override_commit(commit_message)) - return "\n".join(description) - - -def __get_commit_timestamp(commit: Commit) -> int: - """ - # Convert datetime to UTC timestamp. For more info: - # https://stackoverflow.com/questions/5067218/get-utc-timestamp-in-python-with-datetime - - :param commit: a Commit object - :return: the timestamp of the commit - """ - return calendar.timegm(commit.committed_datetime.utctimetuple()) diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py deleted file mode 100755 index fa3062f091..0000000000 --- a/library_generation/generate_repo.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import shutil -from typing import Optional -import library_generation.utils.utilities as util -from library_generation.generate_composed_library import generate_composed_library -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.library_config import LibraryConfig -from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing - - -def generate_from_yaml( - config: GenerationConfig, - repository_path: str, - api_definitions_path: str, - target_library_names: Optional[list[str]], -) -> None: - """ - Based on the generation config, generates libraries via - generate_composed_library.py - - :param config: a GenerationConfig object. - :param repository_path: The repository path to which the generated files - will be sent. - :param api_definitions_path: The path to where the api definition resides. - :param target_library_names: a list of libraries to be generated. - If specified, only the library whose library_name is in target_library_names - will be generated. - If specified with an empty list, then no library will be generated. - If not specified, all libraries in the configuration yaml will be generated. - """ - target_libraries = get_target_libraries( - config=config, target_library_names=target_library_names - ) - repo_config = util.prepare_repo( - gen_config=config, library_config=target_libraries, repo_path=repository_path - ) - # copy api definition to output folder. - shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) - - for library_path, library in repo_config.get_libraries().items(): - print(f"generating library {library.get_library_name()}") - generate_composed_library( - config=config, - library_path=library_path, - library=library, - repo_config=repo_config, - ) - - if not config.is_monorepo() or config.contains_common_protos(): - return - - monorepo_postprocessing( - repository_path=repository_path, versions_file=repo_config.versions_file - ) - - -def get_target_libraries( - config: GenerationConfig, target_library_names: list[str] = None -) -> list[LibraryConfig]: - """ - Returns LibraryConfig objects whose library_name is in target_library_names. - - :param config: a GenerationConfig object. - :param target_library_names: library_name of target libraries. - If not specified, all libraries in the given config will be returned. - :return: LibraryConfig objects. - """ - if target_library_names is None: - return config.libraries - target_libraries = set(target_library_names) - return [ - library - for library in config.libraries - if library.get_library_name() in target_libraries - ] diff --git a/library_generation/model/bom_config.py b/library_generation/model/bom_config.py deleted file mode 100644 index b562407eb7..0000000000 --- a/library_generation/model/bom_config.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class BomConfig: - """ - Class that represents an entry in dependencyManagement section. - """ - - def __init__( - self, - group_id: str, - artifact_id: str, - version: str, - version_annotation: str, - is_import: bool = True, - ): - self.group_id = group_id - self.artifact_id = artifact_id - self.version = version - self.version_annotation = version_annotation - self.is_import = is_import - - def __lt__(self, another): - return self.group_id < another.group_id or ( - self.group_id == another.group_id and self.artifact_id < another.artifact_id - ) - - def __eq__(self, another): - return ( - self.group_id == another.group_id - and self.artifact_id == another.artifact_id - ) diff --git a/library_generation/model/config_change.py b/library_generation/model/config_change.py deleted file mode 100644 index 8a5e813244..0000000000 --- a/library_generation/model/config_change.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import shutil -from enum import Enum -from typing import Optional -from git import Commit, Repo - -from library_generation.model.gapic_inputs import parse_build_str -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.library_config import LibraryConfig -from library_generation.utils.utilities import sh_util -from library_generation.utils.proto_path_utils import find_versioned_proto_path - -INSERTIONS = "insertions" -LINES = "lines" - - -class ChangeType(Enum): - GOOGLEAPIS_COMMIT = 1 - REPO_LEVEL_CHANGE = 2 - LIBRARIES_ADDITION = 3 - # As of Mar. 2024, we decide not to produce this type of change because we - # still need to manually remove the libray. - # LIBRARIES_REMOVAL = 4 - LIBRARY_LEVEL_CHANGE = 5 - GAPIC_ADDITION = 6 - # As of Mar. 2024, we decide not to produce this type of change because we - # still need to manually remove the libray. - # GAPIC_REMOVAL = 7 - - -class HashLibrary: - """ - Data class to group a LibraryConfig object and its hash value together. - """ - - def __init__(self, hash_value: int, library: LibraryConfig): - self.hash_value = hash_value - self.library = library - - -class LibraryChange: - def __init__(self, changed_param: str, current_value: str, library_name: str = ""): - self.changed_param = changed_param - self.current_value = current_value - self.library_name = library_name - - -class QualifiedCommit: - def __init__(self, commit: Commit, libraries: set[str]): - self.commit = commit - self.libraries = libraries - - -class ConfigChange: - ALL_LIBRARIES_CHANGED = None - - def __init__( - self, - change_to_libraries: dict[ChangeType, list[LibraryChange]], - baseline_config: GenerationConfig, - current_config: GenerationConfig, - ): - self.change_to_libraries = change_to_libraries - self.baseline_config = baseline_config - self.current_config = current_config - - def get_changed_libraries(self) -> Optional[list[str]]: - """ - Returns a unique, sorted list of library name of changed libraries. - None if there is a repository level change, which means all libraries - in the current_config will be generated. - - :return: library names of change libraries. - """ - if ChangeType.REPO_LEVEL_CHANGE in self.change_to_libraries: - return ConfigChange.ALL_LIBRARIES_CHANGED - library_names = set() - for change_type, library_changes in self.change_to_libraries.items(): - if change_type == ChangeType.GOOGLEAPIS_COMMIT: - library_names.update(self.__get_library_names_from_qualified_commits()) - else: - library_names.update( - [library_change.library_name for library_change in library_changes] - ) - return sorted(list(library_names)) - - def get_qualified_commits( - self, - repo_url: str = "https://github.com/googleapis/googleapis.git", - ) -> list[QualifiedCommit]: - """ - Returns qualified commits from configuration change. - - A qualified commit is a commit that changes at least one file (excluding - BUILD.bazel) within a versioned proto path in the given proto_paths. - :param repo_url: the repository contains the commit history. - :return: QualifiedCommit objects. - """ - tmp_dir = sh_util("get_output_folder") - shutil.rmtree(tmp_dir, ignore_errors=True) - os.mkdir(tmp_dir) - # we only need commit history, thus shadow clone is enough. - repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"]) - commit = repo.commit(self.current_config.googleapis_commitish) - proto_paths = self.current_config.get_proto_path_to_library_name() - qualified_commits = [] - while str(commit.hexsha) != self.baseline_config.googleapis_commitish: - qualified_commit = ConfigChange.__create_qualified_commit( - proto_paths=proto_paths, commit=commit - ) - if qualified_commit is not None: - qualified_commits.append(qualified_commit) - commit_parents = commit.parents - if len(commit_parents) == 0: - break - commit = commit_parents[0] - shutil.rmtree(tmp_dir, ignore_errors=True) - return qualified_commits - - def __get_library_names_from_qualified_commits(self) -> list[str]: - qualified_commits = self.get_qualified_commits() - library_names = [] - for qualified_commit in qualified_commits: - library_names.extend(qualified_commit.libraries) - return library_names - - @staticmethod - def __create_qualified_commit( - proto_paths: dict[str, str], commit: Commit - ) -> Optional[QualifiedCommit]: - """ - Returns a qualified commit from the given Commit object; otherwise None. - - :param proto_paths: a mapping from versioned proto_path to library_name - :param commit: a GitHub commit object. - :return: qualified commits. - """ - libraries = set() - for file_path, changes in commit.stats.files.items(): - versioned_proto_path = find_versioned_proto_path(file_path) - if versioned_proto_path in proto_paths: - if ( - file_path.endswith("BUILD.bazel") - # Qualify a commit if the commit only added BUILD.bazel - # because it's very unlikely that a commit added BUILD.bazel - # without adding proto files. Therefore, the commit is - # qualified duo to the proto change eventually. - and (not ConfigChange.__is_added(changes)) - and ( - not ConfigChange.__is_qualified_build_change( - commit=commit, build_file_path=file_path - ) - ) - ): - continue - # Even though a commit usually only changes one - # library, we don't want to miss generating a - # library because the commit may change multiple - # libraries. - libraries.add(proto_paths[versioned_proto_path]) - if len(libraries) == 0: - return None - return QualifiedCommit(commit=commit, libraries=libraries) - - @staticmethod - def __is_added(changes: dict[str, int]) -> bool: - return changes[INSERTIONS] == changes[LINES] - - @staticmethod - def __is_qualified_build_change(commit: Commit, build_file_path: str) -> bool: - """ - Checks if the given commit containing a BUILD.bazel change is a - qualified commit. - - The commit is a qualified commit if the - :class:`library_generation.model.gapic_inputs.GapicInputs` objects - parsed from the commit and its parent are different, since there are - changes in fields that used in library generation. - - :param commit: a GitHub commit object. - :param build_file_path: the path of the BUILD.bazel - :return: True if the commit is a qualified commit; False otherwise. - """ - versioned_proto_path = find_versioned_proto_path(build_file_path) - build = str((commit.tree / build_file_path).data_stream.read()) - parent_commit = commit.parents[0] - parent_build = str((parent_commit.tree / build_file_path).data_stream.read()) - inputs = parse_build_str(build, versioned_proto_path) - parent_inputs = parse_build_str(parent_build, versioned_proto_path) - return inputs != parent_inputs diff --git a/library_generation/model/gapic_config.py b/library_generation/model/gapic_config.py deleted file mode 100644 index 3bbba39454..0000000000 --- a/library_generation/model/gapic_config.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re -from typing import Optional - -ALPHA_VERSION = "alpha" -BETA_VERSION = "beta" - - -class GapicConfig: - """ - Class that represents a GAPICs single entry, inside a `LibraryConfig` in - a generation_config.yaml - """ - - def __init__(self, proto_path: str): - self.proto_path = proto_path - self.version = self.__parse_version() - - def is_stable(self): - return ( - self.version - and (ALPHA_VERSION not in self.version) - and (BETA_VERSION not in self.version) - ) - - def get_version(self): - return self.version - - def __parse_version(self) -> Optional[str]: - version_regex = re.compile(r"^v[1-9]+(p[1-9]+)*(alpha|beta)?.*") - for directory in self.proto_path.split("/"): - if version_regex.search(directory): - return directory - return None - - def __eq__(self, other) -> bool: - if not isinstance(other, GapicConfig): - return False - return self.proto_path == other.proto_path - - def __lt__(self, other) -> bool: - if not isinstance(other, GapicConfig): - raise ValueError( - f"Type {type(other)} can't be comparable " f"with GapicConfig." - ) - - self_version = self.get_version() - other_version = other.get_version() - self_dirs = self.proto_path.split("/") - other_dirs = other.proto_path.split("/") - # Case 1: if both of the configs don't have a version in proto_path, - # the one with lower depth is smaller. - if (not self_version) and (not other_version): - return len(self_dirs) < len(other_dirs) - # Case 2: if only one config has a version in proto_path, it is smaller - # than the other one. - if self_version and (not other_version): - return True - if (not self_version) and other_version: - return False - # Two configs both have a version in proto_path. - self_stable = self.is_stable() - other_stable = other.is_stable() - # Case 3, if only config has a stable version in proto_path, it is - # smaller than the other one. - if self_stable and (not other_stable): - return True - if (not self_stable) and other_stable: - return False - # Case 4, if two configs have a non-stable version in proto_path, - # the one with higher version is smaller. - # Note that using string comparison may lead unexpected result, - # e.g., v1beta10 is smaller than v1beta2. - # In reality, however, there's unlikely that a library has >10 beta - # versions. - if (not self_stable) and (not other_stable): - return self_version > other_version - # Two configs both have a stable version in proto_path. - # Case 5, if two configs have different depth in proto_path, the one - # with lower depth is smaller. - if len(self_dirs) != len(other_dirs): - return len(self_dirs) < len(other_dirs) - # Case 6, the config with higher stable version is smaller. - self_num = int(self_version[1:]) - other_num = int(other_version[1:]) - return self_num > other_num diff --git a/library_generation/model/gapic_inputs.py b/library_generation/model/gapic_inputs.py deleted file mode 100644 index 5c07777965..0000000000 --- a/library_generation/model/gapic_inputs.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path -import re - -proto_library_pattern = r""" -proto_library_with_info\( -(.*?) -\) -""" -gapic_pattern = r""" -java_gapic_library\( -(.*?) -\) -""" -assembly_pattern = r""" -java_gapic_assembly_gradle_pkg\( -(.*?) -\) -""" -# match a line which the first character is "#". -comment_pattern = r"^\s*\#+" -pattern_to_proto = { - r"//google/cloud:common_resources_proto": "google/cloud/common_resources.proto", - r"//google/cloud/location:location_proto": "google/cloud/location/locations.proto", - r"//google/iam/v1:iam_policy_proto": "google/iam/v1/iam_policy.proto", -} -transport_pattern = r"transport = \"(.*?)\"" -rest_pattern = r"rest_numeric_enums = True" -gapic_yaml_pattern = r"gapic_yaml = \"(.*?)\"" -service_config_pattern = r"grpc_service_config = \"(.*?)\"" -service_yaml_pattern = r"service_yaml = \"(.*?)\"" -include_samples_pattern = r"include_samples = True" - - -class GapicInputs: - """ - A data class containing inputs to invoke generate_library.sh to generate - a GAPIC library. - """ - - def __init__( - self, - proto_only="true", - additional_protos="google/cloud/common_resources.proto", - transport="grpc", - rest_numeric_enum="", - gapic_yaml="", - service_config="", - service_yaml="", - include_samples="true", - ): - self.proto_only = proto_only - self.additional_protos = additional_protos - self.transport = transport - self.rest_numeric_enum = rest_numeric_enum - self.gapic_yaml = gapic_yaml - self.service_config = service_config - self.service_yaml = service_yaml - self.include_samples = include_samples - - def __eq__(self, other): - if not isinstance(other, GapicInputs): - return False - return ( - self.proto_only == other.proto_only - and self.additional_protos == other.additional_protos - and self.transport == other.transport - and self.rest_numeric_enum == other.rest_numeric_enum - and self.gapic_yaml == other.gapic_yaml - and self.service_config == other.service_config - and self.service_yaml == other.service_yaml - and self.include_samples == other.include_samples - ) - - -def parse( - build_path: Path, versioned_path: str, build_file_name: str = "BUILD.bazel" -) -> GapicInputs: - """ - Utility function to parse inputs of generate_library.sh from BUILD.bazel. - :param build_path: the file path of BUILD.bazel - :param versioned_path: a versioned path in googleapis repository, e.g., - google/cloud/asset/v1. - :param build_file_name: the name of the build file. - :return: an GapicInputs object. - """ - with open(f"{build_path}/{build_file_name}") as build: - content = build.read() - return parse_build_str(build_str=content, versioned_path=versioned_path) - - -def parse_build_str(build_str: str, versioned_path: str) -> GapicInputs: - proto_library_target = re.compile( - proto_library_pattern, re.DOTALL | re.VERBOSE - ).findall(build_str) - additional_protos = "" - if len(proto_library_target) > 0: - additional_protos = __parse_additional_protos(proto_library_target[0]) - gapic_target = re.compile(gapic_pattern, re.DOTALL | re.VERBOSE).findall(build_str) - assembly_target = re.compile(assembly_pattern, re.DOTALL | re.VERBOSE).findall( - build_str - ) - include_samples = "false" - if len(assembly_target) > 0: - include_samples = __parse_include_samples(assembly_target[0]) - if len(gapic_target) == 0: - return GapicInputs( - additional_protos=additional_protos, include_samples=include_samples - ) - - transport = __parse_transport(gapic_target[0]) - rest_numeric_enum = __parse_rest_numeric_enums(gapic_target[0]) - gapic_yaml = __parse_gapic_yaml(gapic_target[0], versioned_path) - service_config = __parse_service_config(gapic_target[0], versioned_path) - service_yaml = __parse_service_yaml(gapic_target[0], versioned_path) - - return GapicInputs( - proto_only="false", - additional_protos=additional_protos, - transport=transport, - rest_numeric_enum=rest_numeric_enum, - gapic_yaml=gapic_yaml, - service_config=service_config, - service_yaml=service_yaml, - include_samples=include_samples, - ) - - -def __parse_additional_protos(proto_library_target: str) -> str: - res = [" "] - lines = proto_library_target.split("\n") - for line in lines: - if len(re.findall(comment_pattern, line)) != 0: - # skip a line which the first charactor is "#" since it's - # a comment. - continue - for pattern in pattern_to_proto: - if len(re.findall(pattern, line)) == 0: - continue - res.append(pattern_to_proto[pattern]) - return " ".join(res) - - -def __parse_transport(gapic_target: str) -> str: - transport = re.findall(transport_pattern, gapic_target) - return transport[0] if len(transport) != 0 else "grpc" - - -def __parse_rest_numeric_enums(gapic_target: str) -> str: - rest_numeric_enums = re.findall(rest_pattern, gapic_target) - return "true" if len(rest_numeric_enums) != 0 else "false" - - -def __parse_gapic_yaml(gapic_target: str, versioned_path: str) -> str: - gapic_yaml = re.findall(gapic_yaml_pattern, gapic_target) - return f"{versioned_path}/{gapic_yaml[0]}" if len(gapic_yaml) != 0 else "" - - -def __parse_service_config(gapic_target: str, versioned_path: str) -> str: - service_config = re.findall(service_config_pattern, gapic_target) - return ( - f"{versioned_path}/{service_config[0]}".replace(":", "") - if len(service_config) != 0 - else "" - ) - - -def __parse_service_yaml(gapic_target: str, versioned_path: str) -> str: - service_yaml = re.findall(service_yaml_pattern, gapic_target) - if len(service_yaml) == 0: - return "" - res = str(service_yaml[0]) - if res.startswith("//"): - # special case if the service config starts with "//", is a Bazel - # target with an absolute path. - return res.replace("//", "").replace(":", "/") - return f"{versioned_path}/{res}" - - -def __parse_include_samples(assembly_target: str) -> str: - include_samples = re.findall(include_samples_pattern, assembly_target) - return "true" if len(include_samples) != 0 else "false" diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py deleted file mode 100644 index bc3ff6d440..0000000000 --- a/library_generation/model/generation_config.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -import yaml -from typing import Optional -from library_generation.model.library_config import LibraryConfig -from library_generation.model.gapic_config import GapicConfig - -REPO_LEVEL_PARAMETER = "Repo level parameter" -LIBRARY_LEVEL_PARAMETER = "Library level parameter" -GAPIC_LEVEL_PARAMETER = "GAPIC level parameter" -COMMON_PROTOS_LIBRARY_NAME = "common-protos" -GAPIC_GENERATOR_VERSION = "gapic_generator_version" -LIBRARIES_BOM_VERSION = "libraries_bom_version" -GENERATOR_VERSION_ENV_KEY = "GENERATOR_VERSION" - - -class GenerationConfig: - """ - Class that represents the root of a generation_config.yaml - """ - - def __init__( - self, - googleapis_commitish: str, - libraries: list[LibraryConfig], - gapic_generator_version: Optional[str] = None, - libraries_bom_version: Optional[str] = None, - grpc_version: Optional[str] = None, - protoc_version: Optional[str] = None, - ): - self.googleapis_commitish = googleapis_commitish - self.libraries_bom_version = ( - libraries_bom_version if libraries_bom_version else "" - ) - self.gapic_generator_version = GenerationConfig.__set_generator_version( - gapic_generator_version - ) - self.libraries = libraries - self.grpc_version = grpc_version - self.protoc_version = protoc_version - # explicit set to None so that we can compute the - # value in getter. - self.__contains_common_protos = None - self.__validate() - - def get_proto_path_to_library_name(self) -> dict[str, str]: - """ - Get versioned proto_path to library_name mapping from configuration. - - :return: versioned proto_path to library_name mapping - """ - paths = {} - for library in self.libraries: - for gapic_config in library.gapic_configs: - paths[gapic_config.proto_path] = library.get_library_name() - return paths - - def is_monorepo(self) -> bool: - return len(self.libraries) > 1 - - def contains_common_protos(self) -> bool: - if self.__contains_common_protos is None: - self.__contains_common_protos = False - for library in self.libraries: - if library.get_library_name() == COMMON_PROTOS_LIBRARY_NAME: - self.__contains_common_protos = True - break - return self.__contains_common_protos - - @staticmethod - def __set_generator_version(gapic_generator_version: Optional[str]) -> str: - if gapic_generator_version is not None: - return gapic_generator_version - # if the generator version is not set through generation config, - # get it from environment variable. - gapic_generator_version = os.getenv(GENERATOR_VERSION_ENV_KEY) - if not gapic_generator_version: - raise ValueError( - f"Environment variable {GENERATOR_VERSION_ENV_KEY}" - f" is not set when the generator version is not" - f" specified in the generation config." - ) - return gapic_generator_version - - def __validate(self) -> None: - seen_library_names = dict() - for library in self.libraries: - library_name = library.get_library_name() - if library_name in seen_library_names: - raise ValueError( - f"Both {library.name_pretty} and " - f"{seen_library_names.get(library_name)} have the same " - f"library name: {library_name}, please update one of the " - f"library to have a different library name." - ) - seen_library_names[library_name] = library.name_pretty - - -def from_yaml(path_to_yaml: str) -> GenerationConfig: - """ - Parses a yaml located in path_to_yaml. - :param path_to_yaml: the path to the configuration file - :return the parsed configuration represented by the "model" classes - """ - with open(path_to_yaml, "r") as file_stream: - config = yaml.safe_load(file_stream) - - libraries = __required(config, "libraries", REPO_LEVEL_PARAMETER) - if not libraries: - raise ValueError(f"Library is None in {path_to_yaml}.") - - parsed_libraries = list() - for library in libraries: - gapics = __required(library, "GAPICs") - - parsed_gapics = list() - if not gapics: - raise ValueError(f"GAPICs is None in {library}.") - for gapic in gapics: - proto_path = __required(gapic, "proto_path", GAPIC_LEVEL_PARAMETER) - new_gapic = GapicConfig(proto_path) - parsed_gapics.append(new_gapic) - - new_library = LibraryConfig( - api_shortname=__required(library, "api_shortname"), - api_description=__required(library, "api_description"), - name_pretty=__required(library, "name_pretty"), - product_documentation=__required(library, "product_documentation"), - gapic_configs=parsed_gapics, - library_type=__optional(library, "library_type", "GAPIC_AUTO"), - release_level=__optional(library, "release_level", "preview"), - api_id=__optional(library, "api_id", None), - api_reference=__optional(library, "api_reference", None), - codeowner_team=__optional(library, "codeowner_team", None), - excluded_poms=__optional(library, "excluded_poms", None), - excluded_dependencies=__optional(library, "excluded_dependencies", None), - client_documentation=__optional(library, "client_documentation", None), - distribution_name=__optional(library, "distribution_name", None), - googleapis_commitish=__optional(library, "googleapis_commitish", None), - group_id=__optional(library, "group_id", "com.google.cloud"), - issue_tracker=__optional(library, "issue_tracker", None), - library_name=__optional(library, "library_name", None), - rest_documentation=__optional(library, "rest_documentation", None), - rpc_documentation=__optional(library, "rpc_documentation", None), - cloud_api=__optional(library, "cloud_api", True), - requires_billing=__optional(library, "requires_billing", True), - extra_versioned_modules=__optional( - library, "extra_versioned_modules", None - ), - recommended_package=__optional(library, "recommended_package", None), - min_java_version=__optional(library, "min_java_version", None), - transport=__optional(library, "transport", None), - ) - parsed_libraries.append(new_library) - - parsed_config = GenerationConfig( - googleapis_commitish=__required( - config, "googleapis_commitish", REPO_LEVEL_PARAMETER - ), - gapic_generator_version=__optional(config, GAPIC_GENERATOR_VERSION, None), - grpc_version=__optional(config, "grpc_version", None), - protoc_version=__optional(config, "protoc_version", None), - libraries_bom_version=__optional(config, LIBRARIES_BOM_VERSION, None), - libraries=parsed_libraries, - ) - - return parsed_config - - -def __required(config: dict, key: str, level: str = LIBRARY_LEVEL_PARAMETER): - if key not in config: - message = ( - f"{level}, {key}, is not found in {config} in yaml." - if level != REPO_LEVEL_PARAMETER - else f"{level}, {key}, is not found in yaml." - ) - raise ValueError(message) - return config[key] - - -def __optional(config: dict, key: str, default: any): - if key not in config: - return default - return config[key] diff --git a/library_generation/model/library_config.py b/library_generation/model/library_config.py deleted file mode 100644 index f7992f47a2..0000000000 --- a/library_generation/model/library_config.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from hashlib import sha256 -from typing import Optional -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.gapic_inputs import GapicInputs - - -MAVEN_COORDINATE_SEPARATOR = ":" - - -class LibraryConfig: - """ - Class that represents a library in a generation_config.yaml file - """ - - def __init__( - self, - api_shortname: str, - api_description: str, - name_pretty: str, - product_documentation: str, - gapic_configs: list[GapicConfig], - library_type: Optional[str] = None, - release_level: Optional[str] = None, - api_id: Optional[str] = None, - api_reference: Optional[str] = None, - codeowner_team: Optional[str] = None, - client_documentation: Optional[str] = None, - distribution_name: Optional[str] = None, - excluded_dependencies: Optional[str] = None, - excluded_poms: Optional[str] = None, - googleapis_commitish: Optional[str] = None, - group_id: Optional[str] = "com.google.cloud", - issue_tracker: Optional[str] = None, - library_name: Optional[str] = None, - rest_documentation: Optional[str] = None, - rpc_documentation: Optional[str] = None, - cloud_api: Optional[bool] = True, - requires_billing: Optional[bool] = True, - extra_versioned_modules: Optional[str] = None, - recommended_package: Optional[str] = None, - min_java_version: Optional[int] = None, - transport: Optional[str] = None, - ): - self.api_shortname = api_shortname - self.api_description = api_description - self.name_pretty = name_pretty - self.product_documentation = product_documentation - self.gapic_configs = gapic_configs - self.library_type = library_type if library_type else "GAPIC_AUTO" - self.release_level = release_level if release_level else "preview" - self.api_id = api_id - self.api_reference = api_reference - self.codeowner_team = codeowner_team - self.excluded_dependencies = excluded_dependencies - self.excluded_poms = excluded_poms - self.client_documentation = client_documentation - self.googleapis_commitish = googleapis_commitish - self.group_id = group_id - self.issue_tracker = issue_tracker - self.library_name = library_name - self.rest_documentation = rest_documentation - self.rpc_documentation = rpc_documentation - self.cloud_api = cloud_api - self.requires_billing = requires_billing - self.extra_versioned_modules = extra_versioned_modules - self.recommended_package = recommended_package - self.min_java_version = min_java_version - self.distribution_name = self.__get_distribution_name(distribution_name) - self.transport = self.__validate_transport(transport) - - def get_library_name(self) -> str: - """ - Return the library name of a given LibraryConfig object - :return: the library name - """ - return self.library_name if self.library_name else self.api_shortname - - def get_sorted_gapic_configs(self) -> list[GapicConfig]: - return sorted(self.gapic_configs) - - def get_maven_coordinate(self) -> str: - """ - Returns the Maven coordinate (group_id:artifact_id) of the library - """ - return self.distribution_name - - def get_artifact_id(self) -> str: - """ - Returns the artifact ID of the library - """ - return self.get_maven_coordinate().split(MAVEN_COORDINATE_SEPARATOR)[-1] - - def get_transport(self, gapic_inputs: GapicInputs) -> str: - """ - Returns the transport of the library. If directly set in library config, return it. - Otherwise, return the transport inferred from gapic_inputs. This value is only - used for postprocessing - the generation still infers the transport from BUILD - files. - """ - return self.transport if self.transport is not None else gapic_inputs.transport - - def __get_distribution_name(self, distribution_name: Optional[str]) -> str: - LibraryConfig.__check_distribution_name(distribution_name) - if distribution_name: - return distribution_name - cloud_prefix = "cloud-" if self.cloud_api else "" - library_name = self.get_library_name() - return f"{self.group_id}:google-{cloud_prefix}{library_name}" - - def __validate_transport(self, transport: str): - if transport not in [None, "grpc", "rest", "both"]: - raise ValueError( - "allowed values for library.transport: grpc, rest, both and None" - ) - return transport - - @staticmethod - def __check_distribution_name(distribution_name: str) -> None: - if not distribution_name: - return - sections = distribution_name.split(MAVEN_COORDINATE_SEPARATOR) - if len(sections) != 2: - raise ValueError(f"{distribution_name} is not a valid distribution name.") - - def __eq__(self, other): - return ( - self.api_shortname == other.api_shortname - and self.api_description == other.api_description - and self.name_pretty == other.name_pretty - and self.product_documentation == other.product_documentation - and self.gapic_configs == other.gapic_configs - and self.library_type == other.library_type - and self.release_level == other.release_level - and self.api_id == other.api_id - and self.api_reference == other.api_reference - and self.codeowner_team == other.codeowner_team - and self.excluded_dependencies == other.excluded_dependencies - and self.excluded_poms == other.excluded_poms - and self.client_documentation == other.client_documentation - and self.distribution_name == other.distribution_name - and self.googleapis_commitish == other.googleapis_commitish - and self.group_id == other.group_id - and self.issue_tracker == other.issue_tracker - and self.library_name == other.library_name - and self.rest_documentation == other.rest_documentation - and self.rpc_documentation == other.rpc_documentation - and self.cloud_api == other.cloud_api - and self.requires_billing == other.requires_billing - and self.extra_versioned_modules == other.extra_versioned_modules - and self.recommended_package == other.recommended_package - and self.min_java_version == other.min_java_version - and self.transport == other.transport - ) - - def __hash__(self): - m = sha256() - m.update( - str( - [ - self.api_shortname, - self.api_description, - self.name_pretty, - self.product_documentation, - self.library_type, - self.release_level, - self.api_id, - self.api_reference, - self.codeowner_team, - self.excluded_dependencies, - self.excluded_poms, - self.client_documentation, - self.distribution_name, - self.googleapis_commitish, - self.group_id, - self.issue_tracker, - self.library_name, - self.rest_documentation, - self.rpc_documentation, - self.cloud_api, - self.requires_billing, - self.extra_versioned_modules, - self.recommended_package, - self.min_java_version, - self.transport, - ] - + [config.proto_path for config in self.gapic_configs] - ).encode("utf-8") - ) - return int(m.hexdigest(), 16) diff --git a/library_generation/model/repo_config.py b/library_generation/model/repo_config.py deleted file mode 100644 index 58ed3fa3bf..0000000000 --- a/library_generation/model/repo_config.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from library_generation.model.library_config import LibraryConfig - - -GRPC_PREFIX = "grpc-" -PROTO_PREFIX = "proto-" -NEW_CLIENT_VERSION = "0.0.0" - - -class RepoConfig: - """ - Class that represents a generated repository - """ - - def __init__( - self, - output_folder: str, - libraries: dict[str, LibraryConfig], - versions_file: str, - ): - """ - Init a RepoConfig object - - :param output_folder: the path to which the generated repo goes - :param libraries: a mapping from library_path to LibraryConfig object - :param versions_file: the path of versions.txt used in post-processing - """ - self.output_folder = output_folder - self.libraries = libraries - self.versions_file = versions_file - self.library_versions = self.__parse_versions() - - def get_libraries(self) -> dict[str, LibraryConfig]: - return self.libraries - - def get_library_version(self, artifact_id: str) -> str: - """ - Returns the version of a given artifact ID. - If the artifact ID is not managed, i.e., a new client, returns `0.0.0`. - - :param artifact_id: the Maven artifact ID. - :return: the version of the artifact. - """ - return self.library_versions.get(artifact_id, NEW_CLIENT_VERSION) - - def __parse_versions(self) -> dict[str, str]: - """ - For a given versions.txt file (defined in self.versions_file) - creates a map of artifact-id to its version - - :return: a map "artifact-id -> version" - """ - library_versions = dict() - with open(self.versions_file) as f: - for line in f.readlines(): - sections = line.split(":") - # skip comments and empty lines. - if len(sections) != 3: - continue - artifact_id = sections[0] - released_version = sections[1] - if artifact_id.startswith(GRPC_PREFIX) or artifact_id.startswith( - PROTO_PREFIX - ): - continue - library_versions[artifact_id] = released_version - return library_versions diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh deleted file mode 100755 index 3d38677678..0000000000 --- a/library_generation/owlbot/bin/entrypoint.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is the entrypoint script for java owlbot. This is not intended to be -# called directly but rather be called from postproces_library.sh -# For reference, the positional arguments are -# 1: scripts_root: location of the root of the library_generation scripts. When -# in a Docker container, this value should be /src/ -# 2: versions_file: points to a versions.txt containing versions to be applied -# both to README and pom.xml files -# 3: is_monorepo: whether we are postprocessing a monorepo -# 4: libraries_bom_version: used to render the readme -# 5: library_version: used to render the readme - -# The scripts assumes the CWD is the folder where postprocessing is going to be -# applied - -set -ex -scripts_root=$1 -versions_file=$2 -is_monorepo=$3 -libraries_bom_version=$4 -library_version=$5 - -if [[ "${is_monorepo}" == "true" ]]; then - mv owl-bot-staging/* temp - rm -rd owl-bot-staging/ - mv temp owl-bot-staging -fi - -# templates as well as retrieving files from owl-bot-staging -echo "Retrieving files from owl-bot-staging directory..." -if [ -f "owlbot.py" ] -then - # we copy the templates to a temp folder because we need to do a special - # modification regarding libraries_bom_version that can't be handled by the - # synthtool library considering the way owlbot.py files are written - export SYNTHTOOL_TEMPLATES="${scripts_root}/owlbot/templates" - export SYNTHTOOL_LIBRARIES_BOM_VERSION="${libraries_bom_version}" - export SYNTHTOOL_LIBRARY_VERSION="${library_version}" - # defaults to run owlbot.py - python3 owlbot.py - unset SYNTHTOOL_TEMPLATES - unset SYNTHTOOL_LIBRARIES_BOM_VERSION - unset SYNTHTOOL_LIBRARY_VERSION -fi -echo "...done" - -# write or restore pom.xml files -echo "Generating missing pom.xml..." -python3 "${scripts_root}/owlbot/src/fix_poms.py" "${versions_file}" "${is_monorepo}" -echo "...done" - -# write or restore clirr-ignored-differences.xml -echo "Generating clirr-ignored-differences.xml..." -"${scripts_root}"/owlbot/bin/write_clirr_ignore.sh "${scripts_root}" -echo "...done" - -# fix license headers -echo "Fixing missing license headers..." -python3 "${scripts_root}/owlbot/src/fix-license-headers.py" -echo "...done" - -echo "Reformatting source..." -"${scripts_root}"/owlbot/bin/format_source.sh "${scripts_root}" -echo "...done" diff --git a/library_generation/owlbot/bin/format_source.sh b/library_generation/owlbot/bin/format_source.sh deleted file mode 100755 index 9402be778b..0000000000 --- a/library_generation/owlbot/bin/format_source.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -# Why OwlBot Java postprocessor does not use the formatter defined in pom.xml? -# It's because the postprocessor runs in a privileged (albeit limited) -# environment. We limit the risk of running somebody else's malicious Maven -# plugin code in the environment. - -# Find all the java files relative to the current directory and format them -# using google-java-format -scripts_root=$1 - -source "${scripts_root}/utils/utilities.sh" -list="$(find . -name '*.java' -not -path ".*/samples/snippets/generated/**/*" )" -tmp_file=$(mktemp) - -for file in $list; -do - if [[ $file =~ .*/samples/snippets/src/main/java/com/example/firestore/Quickstart.java ]]; - then - echo "File skipped formatting: $file" - elif [[ $file =~ .*/samples/snippets/src/.*/java/com/example/spanner/.*.java ]]; - then - echo "File skipped formatting: $file" - else - echo $file >> $tmp_file - fi -done - -# use formatter downloaded in the Dockerfile. -cat $tmp_file | xargs java -jar "$(get_java_formatter_location)" --replace - -rm $tmp_file diff --git a/library_generation/owlbot/bin/write_clirr_ignore.sh b/library_generation/owlbot/bin/write_clirr_ignore.sh deleted file mode 100755 index d6925ef354..0000000000 --- a/library_generation/owlbot/bin/write_clirr_ignore.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -scripts_root=$1 -templates_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")/../templates/clirr) - -# restore default clirr-ignored-differences.xml for protos if the file does not exist -for dir in `ls -d proto-google-*` -do - if [ ! -f "${dir}/clirr-ignored-differences.xml" ] - then - tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX) - pushd ${dir} - pushd src/main/java - find * -name *OrBuilder.java | xargs dirname | sort -u | jq -Rns ' (inputs | rtrimstr("\n") | split("\n") ) as $data | {proto_paths: $data}' > ${tmp_dir}/paths.json - popd - python3 "${scripts_root}/owlbot/src/gen-template.py" --data=${tmp_dir}/paths.json --folder=${templates_dir} - popd - fi -done diff --git a/library_generation/owlbot/src/fix-license-headers.py b/library_generation/owlbot/src/fix-license-headers.py deleted file mode 100644 index 1c690b5d39..0000000000 --- a/library_generation/owlbot/src/fix-license-headers.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path -import glob -from synthtool.languages import java - - -root = Path(".").resolve() - -# Until the generator generates license headers on generated proto -# classes, add the license headers in -for path in glob.glob("proto-*"): - java.fix_proto_headers(root / path) - -# Until the generator generates license headers on generated grpc -# classes, add the license headers in -for path in glob.glob("grpc-*"): - java.fix_grpc_headers(root / path, "unused") diff --git a/library_generation/owlbot/src/fix_poms.py b/library_generation/owlbot/src/fix_poms.py deleted file mode 100644 index 9ee7514a4d..0000000000 --- a/library_generation/owlbot/src/fix_poms.py +++ /dev/null @@ -1,611 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import glob -import json -from xml.etree.ElementTree import ElementTree - -from lxml import etree -import os -import re -from typing import List, Mapping -from library_generation.owlbot.src.poms import module, templates - - -def load_versions(filename: str, default_group_id: str) -> Mapping[str, module.Module]: - if not os.path.isfile(filename): - return {} - modules = {} - with open(filename, "r") as fp: - for line in fp: - line = line.strip() - if line.startswith("#"): - continue - - parts = line.split(":") - if len(parts) == 3: - artifact_id = parts[0] - group_id = ( - # For artifact id starts with `proto-` or `grpc-`, we - # need special treatments to append `.api.grpc` suffix - # to its corresponding group id. - # For other artifact id, keep the existing group id. - # Other than the two aforementioned artifact id, do not - # assume artifact id always starts with `google-`. Known - # exception is ad-manager. - __proto_group_id(default_group_id) - if artifact_id.startswith("proto-") - or artifact_id.startswith("grpc-") - else default_group_id - ) - modules[artifact_id] = module.Module( - group_id=group_id, - artifact_id=artifact_id, - release_version=parts[1], - version=parts[2], - ) - return modules - - -def _find_dependency_index(dependencies, group_id, artifact_id) -> int: - try: - return next( - i - for i, x in enumerate(dependencies.getchildren()) - if _dependency_matches(x, group_id, artifact_id) - ) - except StopIteration: - return -1 - - -def _dependency_matches(node, group_id, artifact_id) -> bool: - artifact_node = node.find("{http://maven.apache.org/POM/4.0.0}artifactId") - group_node = node.find("{http://maven.apache.org/POM/4.0.0}groupId") - - if artifact_node is None or group_node is None: - return False - - return artifact_node.text.startswith(artifact_id) and group_node.text.startswith( - group_id - ) - - -def _is_cloud_client(existing_modules: List[module.Module]) -> bool: - proto_modules_len = 0 - grpc_modules_len = 0 - for artifact in existing_modules: - if artifact.startswith("proto-"): - proto_modules_len += 1 - if artifact.startswith("grpc-"): - grpc_modules_len += 1 - return proto_modules_len > 0 or grpc_modules_len > 0 - - -def update_cloud_pom( - filename: str, - proto_modules: List[module.Module], - grpc_modules: List[module.Module], - is_monorepo: bool, -): - tree = etree.parse(filename) - root = tree.getroot() - dependencies = root.find("{http://maven.apache.org/POM/4.0.0}dependencies") - - existing_dependencies = [ - m.find("{http://maven.apache.org/POM/4.0.0}artifactId").text - for m in dependencies - if m.find("{http://maven.apache.org/POM/4.0.0}artifactId") is not None - ] - - if is_monorepo: - _set_test_scoped_deps(dependencies) - - try: - grpc_index = _find_dependency_index( - dependencies, "com.google.api.grpc", "grpc-" - ) - except StopIteration: - grpc_index = _find_dependency_index(dependencies, "junit", "junit") - # insert grpc dependencies after junit - for m in grpc_modules: - if m.artifact_id not in existing_dependencies: - print(f"adding new test dependency {m.artifact_id}") - new_dependency = etree.Element( - "{http://maven.apache.org/POM/4.0.0}dependency" - ) - new_dependency.tail = "\n " - new_dependency.text = "\n " - new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") - new_group.text = m.group_id - new_group.tail = "\n " - new_artifact = etree.Element( - "{http://maven.apache.org/POM/4.0.0}artifactId" - ) - new_artifact.text = m.artifact_id - new_artifact.tail = "\n " - new_scope = etree.Element("{http://maven.apache.org/POM/4.0.0}scope") - new_scope.text = "test" - new_scope.tail = "\n " - new_dependency.append(new_group) - new_dependency.append(new_artifact) - new_dependency.append(new_scope) - dependencies.insert(grpc_index + 1, new_dependency) - - try: - proto_index = _find_dependency_index( - dependencies, "com.google.api.grpc", "proto-" - ) - except StopIteration: - print("after protobuf") - proto_index = _find_dependency_index( - dependencies, "com.google.protobuf", "protobuf-java" - ) - # insert proto dependencies after protobuf-java - for m in proto_modules: - if m.artifact_id not in existing_dependencies: - if re.match(r"proto-.*-v\d+.*", m.artifact_id): - print(f"adding new dependency {m.artifact_id}") - new_dependency = etree.Element( - "{http://maven.apache.org/POM/4.0.0}dependency" - ) - new_dependency.tail = "\n " - new_dependency.text = "\n " - new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") - new_group.text = m.group_id - new_group.tail = "\n " - new_artifact = etree.Element( - "{http://maven.apache.org/POM/4.0.0}artifactId" - ) - new_artifact.text = m.artifact_id - new_artifact.tail = "\n " - new_dependency.append(new_group) - new_dependency.append(new_artifact) - dependencies.insert(proto_index + 1, new_dependency) - - tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") - - -def _set_test_scoped_deps(dependencies: list[ElementTree]) -> None: - """ - As of July 2024, we have two dependencies that should be declared as - test-scoped in a monorepo: grpc-google-common-protos and grpc-google-iam-v1. - HW libraries are treated as usual - :param dependencies: List of XML Objects representing a - """ - TEST_SCOPED_DEPENDENCIES = ["grpc-google-common-protos", "grpc-google-iam-v1"] - print( - 'converting dependencies "grpc-google-common-protos" and "grpc-google-iam-v1" to test-scoped' - ) - for d in dependencies: - artifact_query = "{http://maven.apache.org/POM/4.0.0}artifactId" - scope_query = "{http://maven.apache.org/POM/4.0.0}scope" - current_scope = d.find(scope_query) - artifact_id_elem = d.find(artifact_query) - if artifact_id_elem is None: - continue - artifact_id = artifact_id_elem.text - is_test_scoped = ( - current_scope.text == "test" if current_scope is not None else False - ) - if artifact_id in TEST_SCOPED_DEPENDENCIES and not is_test_scoped: - new_scope = etree.Element(scope_query) - new_scope.text = "test" - if current_scope is not None: - d.replace(current_scope, new_scope) - else: - d.append(new_scope) - new_scope.tail = "\n " - new_scope.getprevious().tail = "\n " - - -def update_parent_pom(filename: str, modules: List[module.Module]): - tree = etree.parse(filename) - root = tree.getroot() - - # BEGIN: update modules - existing = root.find("{http://maven.apache.org/POM/4.0.0}modules") - - module_names = [m.artifact_id for m in modules] - extra_modules = [ - m.text for i, m in enumerate(existing) if m.text not in module_names - ] - - modules_to_write = module_names + extra_modules - num_modules = len(modules_to_write) - - existing.clear() - existing.text = "\n " - for index, m in enumerate(modules_to_write): - new_module = etree.Element("{http://maven.apache.org/POM/4.0.0}module") - new_module.text = m - if index == num_modules - 1: - new_module.tail = "\n " - else: - new_module.tail = "\n " - existing.append(new_module) - - existing.tail = "\n\n " - # END: update modules - - # BEGIN: update versions in dependencyManagement - dependencies = root.find( - "{http://maven.apache.org/POM/4.0.0}dependencyManagement" - ).find("{http://maven.apache.org/POM/4.0.0}dependencies") - - existing_dependencies = [ - m.find("{http://maven.apache.org/POM/4.0.0}artifactId").text - for m in dependencies - if m.find("{http://maven.apache.org/POM/4.0.0}artifactId") is not None - ] - insert_index = 1 - - num_modules = len(modules) - - for index, m in enumerate(modules): - if m.artifact_id in existing_dependencies: - continue - - new_dependency = etree.Element("{http://maven.apache.org/POM/4.0.0}dependency") - new_dependency.tail = "\n " - new_dependency.text = "\n " - new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") - new_group.text = m.group_id - new_group.tail = "\n " - new_artifact = etree.Element("{http://maven.apache.org/POM/4.0.0}artifactId") - new_artifact.text = m.artifact_id - new_artifact.tail = "\n " - new_version = etree.Element("{http://maven.apache.org/POM/4.0.0}version") - new_version.text = m.version - comment = etree.Comment(" {x-version-update:" + m.artifact_id + ":current} ") - comment.tail = "\n " - new_dependency.append(new_group) - new_dependency.append(new_artifact) - new_dependency.append(new_version) - new_dependency.append(comment) - new_dependency.tail = "\n " - dependencies.insert(1, new_dependency) - - # END: update versions in dependencyManagement - - tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") - - -def update_bom_pom(filename: str, modules: List[module.Module]): - tree = etree.parse(filename) - root = tree.getroot() - existing = root.find( - "{http://maven.apache.org/POM/4.0.0}dependencyManagement" - ).find("{http://maven.apache.org/POM/4.0.0}dependencies") - - num_modules = len(modules) - - existing.clear() - existing.text = "\n " - for index, m in enumerate(modules): - new_dependency = etree.Element("{http://maven.apache.org/POM/4.0.0}dependency") - new_dependency.tail = "\n " - new_dependency.text = "\n " - new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") - new_group.text = m.group_id - new_group.tail = "\n " - new_artifact = etree.Element("{http://maven.apache.org/POM/4.0.0}artifactId") - new_artifact.text = m.artifact_id - new_artifact.tail = "\n " - new_version = etree.Element("{http://maven.apache.org/POM/4.0.0}version") - new_version.text = m.version - comment = etree.Comment(" {x-version-update:" + m.artifact_id + ":current} ") - comment.tail = "\n " - new_dependency.append(new_group) - new_dependency.append(new_artifact) - new_dependency.append(new_version) - new_dependency.append(comment) - - if index == num_modules - 1: - new_dependency.tail = "\n " - else: - new_dependency.tail = "\n " - existing.append(new_dependency) - - existing.tail = "\n " - - tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") - - -# When generating non-cloud client library, the group id of proto/grpc artifacts -# is prefixed with `{main_artifact_group_id}.api.grpc`, rather than -# `com.google.api.grpc`. -# https://github.com/googleapis/google-cloud-java/issues/9125 -# However, some exceptions are com.google.area120 and com.google.analytics. -# https://github.com/googleapis/google-cloud-java/issues/9304 -def __proto_group_id(main_artifact_group_id: str) -> str: - prefix = "com.google" - list_of_group_id = [ - "com.google.cloud", - "com.google.area120", - "com.google.analytics", - ] - if main_artifact_group_id not in list_of_group_id: - prefix = main_artifact_group_id - return f"{prefix}.api.grpc" - - -def __get_monorepo_version(versions: str) -> str: - """ - Returns the current version of google-cloud-java in the given version file - :param versions: the versions.txt - :return: the current version of google-cloud-java - """ - with open(versions, "r") as f: - for line in f.readlines(): - if "google-cloud-java" in line: - return line.split(":")[-1].strip() - - -def main(versions_file, monorepo): - print(f"working directory: {os.getcwd()}") - with open(".repo-metadata.json", "r") as fp: - repo_metadata = json.load(fp) - group_id, artifact_id = repo_metadata["distribution_name"].split(":") - name = repo_metadata["name_pretty"] - existing_modules = load_versions(versions_file, group_id) - print(f"monorepo? {monorepo}") - - # extra modules that need to be manages in versions.txt - if "extra_versioned_modules" in repo_metadata: - extra_managed_modules = repo_metadata["extra_versioned_modules"].split(",") - else: - extra_managed_modules = "" - - # list of modules to be excluded from added to poms - if "excluded_dependencies" in repo_metadata: - excluded_dependencies_list = repo_metadata["excluded_dependencies"].split(",") - else: - excluded_dependencies_list = "" - - # list of poms that have to be excluded from post processing - if "excluded_poms" in repo_metadata: - excluded_poms_list = repo_metadata["excluded_poms"].split(",") - else: - excluded_poms_list = "" - - # Missing Case 1: When this library ('java-XXX' module) is new. - if artifact_id not in existing_modules: - existing_modules[artifact_id] = module.Module( - group_id=group_id, - artifact_id=artifact_id, - version="0.0.1-SNAPSHOT", - release_version="0.0.0", - ) - main_module = existing_modules[artifact_id] - - # Artifact ID is part of distribution name field in .repo-metadata.json - if artifact_id in [ - "grafeas", - "google-cloud-dns", - "google-cloud-notification", - "google-iam-policy", - ]: - # There are special libraries that are not automatically generated - print( - f"Skipping a special case library {artifact_id} that do not have " - " the standard module structure." - ) - return - - parent_artifact_id = f"{artifact_id}-parent" - - if parent_artifact_id not in existing_modules: - existing_modules[parent_artifact_id] = module.Module( - group_id=group_id, - artifact_id=parent_artifact_id, - version=main_module.version, - release_version=main_module.release_version, - ) - parent_module = existing_modules[parent_artifact_id] - - required_dependencies = {} - for dependency_module in existing_modules: - if dependency_module in excluded_dependencies_list: - continue - dep_artifact_id = existing_modules[dependency_module].artifact_id - if monorepo and not os.path.isdir(dep_artifact_id): - # In monorepo, existing_modules are loaded from the root - # versions.txt and thus includes irrelevant artifacts - continue - required_dependencies[dependency_module] = existing_modules[dependency_module] - - # Missing Case 2: There's a new proto-XXX and grpc-XXX directory. It's a new - # version in the proto file to a library. Both a new library and existing - # library. - for path in glob.glob("proto-*"): - if not path in existing_modules: - existing_modules[path] = module.Module( - group_id=__proto_group_id(group_id), - artifact_id=path, - version=main_module.version, - release_version=main_module.release_version, - ) - if ( - path not in excluded_dependencies_list - and path not in main_module.artifact_id - ): - required_dependencies[path] = module.Module( - group_id=__proto_group_id(group_id), - artifact_id=path, - version=main_module.version, - release_version=main_module.release_version, - ) - if not os.path.isfile(f"{path}/pom.xml"): - print(f"creating missing proto pom: {path}") - templates.render( - template_name="proto_pom.xml.j2", - output_name=f"{path}/pom.xml", - module=required_dependencies[path], - parent_module=parent_module, - main_module=main_module, - monorepo=monorepo, - ) - if ( - path not in excluded_dependencies_list - and path not in main_module.artifact_id - ): - required_dependencies[path] = module.Module( - group_id=__proto_group_id(group_id), - artifact_id=path, - version=main_module.version, - release_version=main_module.release_version, - ) - - for path in glob.glob("grpc-*"): - if not path in existing_modules: - existing_modules[path] = module.Module( - group_id=__proto_group_id(group_id), - artifact_id=path, - version=main_module.version, - release_version=main_module.release_version, - ) - if ( - path not in excluded_dependencies_list - and path not in main_module.artifact_id - ): - required_dependencies[path] = module.Module( - group_id=__proto_group_id(group_id), - artifact_id=path, - version=main_module.version, - release_version=main_module.release_version, - ) - - if not os.path.isfile(f"{path}/pom.xml"): - proto_artifact_id = path.replace("grpc-", "proto-") - print(f"creating missing grpc pom: {path}") - templates.render( - template_name="grpc_pom.xml.j2", - output_name=f"{path}/pom.xml", - module=required_dependencies[path], - parent_module=parent_module, - main_module=main_module, - proto_module=existing_modules[proto_artifact_id], - monorepo=monorepo, - ) - if ( - path not in excluded_dependencies_list - and path not in main_module.artifact_id - ): - required_dependencies[path] = module.Module( - group_id=__proto_group_id(group_id), - artifact_id=path, - version=main_module.version, - release_version=main_module.release_version, - ) - proto_modules = [ - module - for module in required_dependencies.values() - if module.artifact_id.startswith("proto-") - and module.artifact_id not in parent_artifact_id - ] - grpc_modules = [ - module - for module in required_dependencies.values() - if module.artifact_id.startswith("grpc-") - and module.artifact_id not in parent_artifact_id - ] - if main_module in grpc_modules or main_module in proto_modules: - modules = grpc_modules + proto_modules - else: - modules = [main_module] + grpc_modules + proto_modules - - if not _is_cloud_client(existing_modules): - print("no proto or grpc modules - probably not a cloud client") - return - - if os.path.isfile(f"{artifact_id}/pom.xml"): - print("updating modules in cloud pom.xml") - if artifact_id not in excluded_poms_list: - update_cloud_pom( - f"{artifact_id}/pom.xml", proto_modules, grpc_modules, monorepo - ) - elif artifact_id not in excluded_poms_list: - print("creating missing cloud pom.xml") - templates.render( - template_name="cloud_pom.xml.j2", - output_name=f"{artifact_id}/pom.xml", - module=main_module, - parent_module=parent_module, - repo=repo_metadata["repo"], - name=name, - description=repo_metadata["api_description"], - proto_modules=proto_modules, - grpc_modules=grpc_modules, - monorepo=monorepo, - ) - - if os.path.isfile(f"{artifact_id}-bom/pom.xml"): - print("updating modules in bom pom.xml") - if artifact_id + "-bom" not in excluded_poms_list: - update_bom_pom(f"{artifact_id}-bom/pom.xml", modules) - elif artifact_id + "-bom" not in excluded_poms_list: - print("creating missing bom pom.xml") - monorepo_version = __get_monorepo_version(versions_file) if monorepo else "" - templates.render( - template_name="bom_pom.xml.j2", - output_name=f"{artifact_id}-bom/pom.xml", - repo=repo_metadata["repo"], - name=name, - modules=modules, - main_module=main_module, - monorepo=monorepo, - monorepo_version=monorepo_version, - ) - - if os.path.isfile("pom.xml"): - print("updating modules in parent pom.xml") - update_parent_pom("pom.xml", modules) - else: - print("creating missing parent pom.xml") - monorepo_version = __get_monorepo_version(versions_file) if monorepo else "" - templates.render( - template_name="parent_pom.xml.j2", - output_name="./pom.xml", - repo=repo_metadata["repo"], - modules=modules, - main_module=main_module, - name=name, - monorepo=monorepo, - monorepo_version=monorepo_version, - ) - - print(f"updating modules in {versions_file}") - existing_modules.pop(parent_artifact_id) - - # add extra modules to versions.txt - for dependency_module in extra_managed_modules: - if dependency_module not in existing_modules: - existing_modules[dependency_module] = module.Module( - group_id=__proto_group_id(group_id), - artifact_id=dependency_module, - version=main_module.version, - release_version=main_module.release_version, - ) - templates.render( - template_name="versions.txt.j2", - output_name=versions_file, - modules=existing_modules.values(), - ) - - -if __name__ == "__main__": - versions_file = sys.argv[1] - monorepo = True if sys.argv[2].lower() == "true" else False - main(versions_file, monorepo) diff --git a/library_generation/owlbot/src/gen-template.py b/library_generation/owlbot/src/gen-template.py deleted file mode 100644 index 30f8c43529..0000000000 --- a/library_generation/owlbot/src/gen-template.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import glob -import json -from typing import List -import os -from pathlib import Path - -import click -import jinja2 - - -@click.command() -@click.option( - "--folder", - help="Path to folder of templates", -) -@click.option("--file", help="Path to template file") -@click.option( - "--data", - help="Path to JSON file with template values", - multiple=True, - required=True, -) -@click.option( - "--output", - help="Path to output", - default=".", -) -def main(folder: str, file: str, data: List[str], output: str): - """Generate templates""" - variables = {} - for data_file in data: - with open(data_file, "r") as fp: - variables = {**variables, **json.load(fp)} - - if folder is not None: - location = Path(folder) - filenames = glob.glob(f"{folder}/**/*.j2", recursive=True) - elif file is not None: - location = Path(file).parent - filenames = [f"{file}.j2"] - else: - raise Exception("Need to specify either folder or file") - - output_path = Path(output) - - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(str(location)), - autoescape=False, - keep_trailing_newline=True, - ) - - for filename in filenames: - template_name = Path(filename).relative_to(location) - template = env.get_template(str(template_name)) - output = template.stream(**variables) - - destination = output_path / os.path.splitext(template_name)[0] - destination.parent.mkdir(parents=True, exist_ok=True) - - with destination.open("w") as fp: - output.dump(fp) - - # Copy file mode over - source_path = Path(template.filename) - mode = source_path.stat().st_mode - destination.chmod(mode) - - -if __name__ == "__main__": - main() diff --git a/library_generation/owlbot/src/poms/.gitignore b/library_generation/owlbot/src/poms/.gitignore deleted file mode 100644 index c18dd8d83c..0000000000 --- a/library_generation/owlbot/src/poms/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__/ diff --git a/library_generation/owlbot/src/poms/module.py b/library_generation/owlbot/src/poms/module.py deleted file mode 100644 index b4274ea969..0000000000 --- a/library_generation/owlbot/src/poms/module.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import attr -from lxml import etree -import os -from typing import List, Optional - - -@attr.s(auto_attribs=True) -class Module: - group_id: str - artifact_id: str - version: str - release_version: Optional[str] - - -def read_module(pom: str) -> Module: - tree = etree.parse(pom) - artifact_id = tree.find("{http://maven.apache.org/POM/4.0.0}artifactId").text - version = tree.find("{http://maven.apache.org/POM/4.0.0}version").text - group_id = ( - "com.google.cloud" - if artifact_id.startswith("google-cloud") - else "com.google.api.grpc" - ) - return Module( - group_id=group_id, - artifact_id=artifact_id, - version=version, - ) - - -def read_modules(service: str) -> List[Module]: - thedir = f"workspace/java-{service}/" - modules = [] - for name in os.listdir(thedir): - dir = os.path.join(thedir, name) - pom = os.path.join(dir, "pom.xml") - if os.path.exists(pom): - modules.append(read_module(pom)) - - return modules diff --git a/library_generation/owlbot/src/poms/templates.py b/library_generation/owlbot/src/poms/templates.py deleted file mode 100644 index 09a77e9ec7..0000000000 --- a/library_generation/owlbot/src/poms/templates.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from jinja2 import Environment, FileSystemLoader -import os -import pathlib -from typing import List - -root_directory = pathlib.Path( - os.path.realpath(os.path.dirname(os.path.realpath(__file__))) -).parent.parent -print(root_directory) -jinja_env = Environment( - loader=FileSystemLoader(str(root_directory / "templates" / "poms")), - keep_trailing_newline=True, -) - - -def render(template_name: str, output_name: str, **kwargs): - template = jinja_env.get_template(template_name) - t = template.stream(kwargs) - directory = os.path.dirname(output_name) - if not os.path.isdir(directory): - os.makedirs(directory) - t.dump(str(output_name)) diff --git a/library_generation/owlbot/synthtool/__init__.py b/library_generation/owlbot/synthtool/__init__.py deleted file mode 100644 index e2d44deaf0..0000000000 --- a/library_generation/owlbot/synthtool/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Synthtool synthesizes libraries from disparate sources.""" - -from synthtool.transforms import ( - move, - replace, - get_staging_dirs, - remove_staging_dirs, -) - -copy = move - -__all__ = [ - "copy", - "move", - "replace", - "get_staging_dirs", - "remove_staging_dirs", -] diff --git a/library_generation/owlbot/synthtool/_tracked_paths.py b/library_generation/owlbot/synthtool/_tracked_paths.py deleted file mode 100644 index b72d7037fe..0000000000 --- a/library_generation/owlbot/synthtool/_tracked_paths.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tracked paths. - -This is a bit of a hack (imported from original synthtool library). -""" - -import pathlib - - -_tracked_paths = [] - - -def add(path): - _tracked_paths.append(pathlib.Path(path)) - # Reverse sort the list, so that the deepest paths get matched first. - _tracked_paths.sort(key=lambda s: -len(str(s))) - - -def relativize(path): - path = pathlib.Path(path) - for tracked_path in _tracked_paths: - try: - return path.relative_to(tracked_path) - except ValueError: - pass - raise ValueError(f"The root for {path} is not tracked.") diff --git a/library_generation/owlbot/synthtool/gcp/common.py b/library_generation/owlbot/synthtool/gcp/common.py deleted file mode 100644 index 5cd648be36..0000000000 --- a/library_generation/owlbot/synthtool/gcp/common.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import sys -import yaml -from pathlib import Path -from typing import Dict, List, Optional -import logging - -from synthtool import _tracked_paths -from synthtool.sources import templates - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) -# Originally brought from gcp/partials.py. -# These are the default locations to look up -_DEFAULT_PARTIAL_FILES = [ - ".readme-partials.yml", - ".readme-partials.yaml", - ".integration-partials.yaml", -] - - -class CommonTemplates: - def __init__(self, template_path: Optional[Path] = None): - local_templates = os.environ.get("SYNTHTOOL_TEMPLATES") - if local_templates is None: - logger.error("env var SYNTHTOOL_TEMPLATES must be set") - sys.exit(1) - logger.debug(f"Using local templates at {local_templates}") - self._template_root = Path(local_templates) - self._templates = templates.Templates(self._template_root) - self.excludes = [] # type: List[str] - - def _generic_library(self, directory: str, relative_dir=None, **kwargs) -> Path: - # load common repo meta information (metadata that's not language specific). - if "metadata" in kwargs: - self._load_generic_metadata(kwargs["metadata"], relative_dir=relative_dir) - # if no samples were found, don't attempt to render a - # samples/README.md. - if "samples" not in kwargs["metadata"] or not kwargs["metadata"]["samples"]: - self.excludes.append("samples/README.md") - - t = templates.TemplateGroup(self._template_root / directory, self.excludes) - - result = t.render(**kwargs) - _tracked_paths.add(result) - - return result - - def java_library(self, **kwargs) -> Path: - # kwargs["metadata"] is required to load values from .repo-metadata.json - if "metadata" not in kwargs: - kwargs["metadata"] = {} - return self._generic_library("java_library", **kwargs) - - def render(self, template_name: str, **kwargs) -> Path: - template = self._templates.render(template_name, **kwargs) - _tracked_paths.add(template) - return template - - def _load_generic_metadata(self, metadata: Dict, relative_dir=None): - """ - loads additional meta information from .repo-metadata.json. - """ - metadata["partials"] = load_partials() - - # Loads repo metadata information from the default location if it - # hasn't already been set. Some callers may have already loaded repo - # metadata, so we don't need to do it again or overwrite it. Also, only - # set the "repo" key. - if "repo" not in metadata: - metadata["repo"] = _load_repo_metadata(relative_dir=relative_dir) - - -def _load_repo_metadata( - relative_dir=None, metadata_file: str = "./.repo-metadata.json" -) -> Dict: - """Parse a metadata JSON file into a Dict. - - Currently, the defined fields are: - * `name` - The service's API name - * `name_pretty` - The service's API title. This will be used for generating titles on READMEs - * `product_documentation` - The product documentation on cloud.google.com - * `client_documentation` - The client library reference documentation - * `issue_tracker` - The public issue tracker for the product - * `release_level` - The release level of the client library. One of: alpha, beta, - ga, deprecated, preview, stable - * `language` - The repo language. One of dotnet, go, java, nodejs, php, python, ruby - * `repo` - The GitHub repo in the format {owner}/{repo} - * `distribution_name` - The language-idiomatic package/distribution name - * `api_id` - The API ID associated with the service. Fully qualified identifier use to - enable a service in the cloud platform (e.g. monitoring.googleapis.com) - * `requires_billing` - Whether or not the API requires billing to be configured on the - customer's acocunt - - Args: - metadata_file (str, optional): Path to the metadata json file - - Returns: - A dictionary of metadata. This may not necessarily include all the defined fields above. - """ - if relative_dir is not None: - if os.path.exists(Path(relative_dir, metadata_file).resolve()): - with open(Path(relative_dir, metadata_file).resolve()) as f: - return json.load(f) - elif os.path.exists(metadata_file): - with open(metadata_file) as f: - return json.load(f) - return {} - - -def load_partials(files: List[str] = []) -> Dict: - """ - hand-crafted artisanal markdown can be provided in a .readme-partials.yml. - The following fields are currently supported: - - body: custom body to include in the usage section of the document. - samples_body: an optional body to place below the table of contents - in samples/README.md. - introduction: a more thorough introduction than metadata["description"]. - title: provide markdown to use as a custom title. - deprecation_warning: a warning to indicate that the library has been - deprecated and a pointer to an alternate option - """ - result: Dict[str, Dict] = {} - cwd_path = Path(os.getcwd()) - for file in files + _DEFAULT_PARTIAL_FILES: - partials_file = cwd_path / file - if os.path.exists(partials_file): - with open(partials_file) as f: - result.update(yaml.load(f, Loader=yaml.SafeLoader)) - return result diff --git a/library_generation/owlbot/synthtool/gcp/samples.py b/library_generation/owlbot/synthtool/gcp/samples.py deleted file mode 100644 index 760fc06574..0000000000 --- a/library_generation/owlbot/synthtool/gcp/samples.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import glob -import logging -import re -import os -import yaml -from typing import List, Dict - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - - -def _read_sample_metadata_comment(sample_file: str) -> Dict: - """Additional meta-information can be provided through embedded comments: - - // sample-metadata: - // title: ACL (Access Control) - // description: Demonstrates setting access control rules. - // usage: node iam.js --help - """ - sample_metadata = {} # type: Dict[str, str] - with open(sample_file) as f: - contents = f.read() - match = re.search( - r"(?P// *sample-metadata:([^\n]+|\n//)+)", contents, re.DOTALL - ) - if match: - # the metadata yaml is stored in a comments, remove the - # prefix so that we can parse the yaml contained. - sample_metadata_string = re.sub(r"((#|//) ?)", "", match.group("metadata")) - try: - sample_metadata = yaml.load( - sample_metadata_string, Loader=yaml.SafeLoader - )["sample-metadata"] - except yaml.scanner.ScannerError: - # warn and continue on bad metadata - logger.warning(f"bad metadata detected in {sample_file}") - return sample_metadata - - -def _sample_metadata(file: str) -> Dict[str, str]: - metadata = { - "title": _decamelize(os.path.splitext(os.path.basename(file))[0]), - "file": file, - } - return {**metadata, **_read_sample_metadata_comment(file)} - - -def all_samples(sample_globs: List[str]) -> List[Dict[str, str]]: - """Walks samples directory and builds up samples data-structure - - Args: - sample_globs: (List[str]): List of path globs to search for samples - - Returns: - A list of sample metadata in the format: - { - "title": "Requester Pays", - "file": "samples/requesterPays.js" - } - The file path is the relative path from the repository root. - """ - files = [] - for sample_glob in sample_globs: - for file in glob.glob(sample_glob, recursive=True): - files.append(file) - return [_sample_metadata(file) for file in sorted(files)] - - -def _decamelize(value: str): - """Parser to convert fooBar.js to Foo Bar.""" - if not value: - return "" - str_decamelize = re.sub("^.", value[0].upper(), value) # apple -> Apple. - str_decamelize = re.sub( - "([A-Z]+)([A-Z])([a-z0-9])", r"\1 \2\3", str_decamelize - ) # ACLBatman -> ACL Batman. - return re.sub("([a-z0-9])([A-Z])", r"\1 \2", str_decamelize) # FooBar -> Foo Bar. diff --git a/library_generation/owlbot/synthtool/gcp/snippets.py b/library_generation/owlbot/synthtool/gcp/snippets.py deleted file mode 100644 index 5db00656b5..0000000000 --- a/library_generation/owlbot/synthtool/gcp/snippets.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import glob -import os -import re -from typing import Dict, List - -OPEN_SNIPPET_REGEX = r".*\[START ([a-z0-9_]+)\].*$" -CLOSE_SNIPPET_REGEX = r".*\[END ([a-z0-9_]+)\].*$" -OPEN_EXCLUDE_REGEX = r".*\[START_EXCLUDE\].*$" -CLOSE_EXCLUDE_REGEX = r".*\[END_EXCLUDE\].*$" - - -def _trim_leading_whitespace(lines: List[str]) -> List[str]: - """Trims leading, plain spaces from the snippet content. Finds the minimum - number of leading spaces, ignoring empty lines, and removes that number of - spaces from each line. - - Args: - lines (List[str]): Lines of content. These lines are newline terminated. - - Returns: - List of trimmed lines. - """ - - def number_of_leading_spaces(input: str) -> int: - return len(input) - len(input.lstrip(" ")) - - def is_empty_line(input: str) -> bool: - if re.match(r"^\s*$", input): - return True - return False - - leading_spaces = [ - number_of_leading_spaces(line) for line in lines if not is_empty_line(line) - ] - max_leading_spaces = min(leading_spaces) - return [ - "\n" if is_empty_line(line) else line[max_leading_spaces:] for line in lines - ] - - -def all_snippets_from_file(sample_file: str) -> Dict[str, str]: - """Reads in a sample file and parse out all contained snippets. - - Args: - sample_file (str): Sample file to parse. - - Returns: - Dictionary of snippet name to snippet code. - """ - if not os.path.exists(sample_file): - return {} - - snippet_lines = {} # type: Dict[str, List[str]] - open_snippets = set() - with open(sample_file) as f: - excluding = False - # Iterate over each line: - # - If the line matches an opening snippet tag, add that snippet tag to - # the set of open tags. - # - If the line matches a closing snippet tag, remove that snippet tag - # from the set of open tags. - # - If the line matches an opening exclude tag, record that we excluding - # content. - # - If the line matches a closing exclude tag, record that we are capturing - # content again. - # - Otherwise, if we are not excluding content, add the line to each of the - # open snippets - # - # This allows us to handle parsing nested or interleaved snippets and ignore - # blocks of code in the snippets - for line in f: - open_match = re.match(pattern=OPEN_SNIPPET_REGEX, string=line) - close_match = re.match(pattern=CLOSE_SNIPPET_REGEX, string=line) - open_exclude_match = re.match(pattern=OPEN_EXCLUDE_REGEX, string=line) - close_exclude_match = re.match(pattern=CLOSE_EXCLUDE_REGEX, string=line) - if open_match and not excluding: - open_snippets.add(open_match[1]) - if not open_match[1] in snippet_lines: - snippet_lines[open_match[1]] = [] - elif close_match and not excluding: - open_snippets.discard(close_match[1]) - elif open_exclude_match: - excluding = True - elif close_exclude_match: - excluding = False - elif not excluding: - for snippet in open_snippets: - snippet_lines[snippet].append(line) - - return { - snippet: "".join(_trim_leading_whitespace(lines)) - for snippet, lines in snippet_lines.items() - } - - -def all_snippets(snippet_globs: List[str]) -> Dict[str, str]: - """Walks the samples directory and parses snippets from each file. - - Args: - snippet_globs (List[str]): List of path globs to expand. - - Returns: - Dictionary of snippet name to snippet code. - """ - snippets = {} - for snippet_glob in snippet_globs: - for file in glob.glob(snippet_glob, recursive=True): - for snippet, code in all_snippets_from_file(file).items(): - snippets[snippet] = code - return snippets diff --git a/library_generation/owlbot/synthtool/languages/java.py b/library_generation/owlbot/synthtool/languages/java.py deleted file mode 100644 index 3b8d321937..0000000000 --- a/library_generation/owlbot/synthtool/languages/java.py +++ /dev/null @@ -1,611 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import xml.etree.ElementTree as ET -import re -import requests -import yaml -import synthtool as s -import synthtool.gcp as gcp -import logging -from synthtool.gcp import common, samples, snippets -from pathlib import Path -from typing import Any, Optional, Dict, Iterable, List -from datetime import date - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - - -JAR_DOWNLOAD_URL = "https://github.com/google/google-java-format/releases/download/google-java-format-{version}/google-java-format-{version}-all-deps.jar" -DEFAULT_FORMAT_VERSION = "1.7" -CURRENT_YEAR = date.today().year -GOOD_LICENSE = f"""/* - * Copyright {CURRENT_YEAR} Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -""" -PROTOBUF_HEADER = "// Generated by the protocol buffer compiler. DO NOT EDIT!" -BAD_LICENSE = """/\\* - \\* Copyright \\d{4} Google LLC - \\* - \\* Licensed under the Apache License, Version 2.0 \\(the "License"\\); you may not use this file except - \\* in compliance with the License. You may obtain a copy of the License at - \\* - \\* http://www.apache.org/licenses/LICENSE-2.0 - \\* - \\* Unless required by applicable law or agreed to in writing, software distributed under the License - \\* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - \\* or implied. See the License for the specific language governing permissions and limitations under - \\* the License. - \\*/ -""" -DEFAULT_MIN_SUPPORTED_JAVA_VERSION = 8 -METADATA = "metadata" -LIBRARIES_BOM_VERSION = "libraries_bom_version" -LIBRARIES_BOM_VERSION_ENV_KEY = "SYNTHTOOL_LIBRARIES_BOM_VERSION" -HEADER_REGEX = re.compile("\\* Copyright \\d{4} Google LLC") -LIBRARY_VERSION = "library_version" -LIBRARY_VERSION_ENV_KEY = "SYNTHTOOL_LIBRARY_VERSION" - - -def _file_has_header(path: Path) -> bool: - """Return true if the file already contains a license header.""" - with open(path, "rt") as fp: - for line in fp: - if HEADER_REGEX.search(line): - return True - return False - - -def _filter_no_header(paths: Iterable[Path]) -> Iterable[Path]: - """Return a subset of files that do not already have a header.""" - for path in paths: - anchor = Path(path.anchor) - remainder = str(path.relative_to(path.anchor)) - for file in anchor.glob(remainder): - if not _file_has_header(file): - yield file - - -def fix_proto_headers(proto_root: Path) -> None: - """Helper to ensure that generated proto classes have appropriate license headers. - - If the file does not already contain a license header, inject one at the top of the file. - Some resource name classes may contain malformed license headers. In those cases, replace - those with our standard license header. - """ - s.replace( - _filter_no_header([proto_root / "src/**/*.java"]), - PROTOBUF_HEADER, - f"{GOOD_LICENSE}{PROTOBUF_HEADER}", - ) - # https://github.com/googleapis/gapic-generator/issues/3074 - s.replace( - [proto_root / "src/**/*Name.java", proto_root / "src/**/*Names.java"], - BAD_LICENSE, - GOOD_LICENSE, - ) - - -def fix_grpc_headers(grpc_root: Path, package_name: str = "unused") -> None: - """Helper to ensure that generated grpc stub classes have appropriate license headers. - - If the file does not already contain a license header, inject one at the top of the file. - """ - s.replace( - _filter_no_header([grpc_root / "src/**/*.java"]), - "^package (.*);", - f"{GOOD_LICENSE}package \\1;", - ) - - -def latest_maven_version(group_id: str, artifact_id: str) -> Optional[str]: - """Helper function to find the latest released version of a Maven artifact. - - Fetches metadata from Maven Central and parses out the latest released - version. - - Args: - group_id (str): The groupId of the Maven artifact - artifact_id (str): The artifactId of the Maven artifact - - Returns: - The latest version of the artifact as a string or None - """ - group_path = "/".join(group_id.split(".")) - url = ( - f"https://repo1.maven.org/maven2/{group_path}/{artifact_id}/maven-metadata.xml" - ) - response = requests.get(url) - if response.status_code >= 400: - return "0.0.0" - - return version_from_maven_metadata(response.text) - - -def version_from_maven_metadata(metadata: str) -> Optional[str]: - """Helper function to parse the latest released version from the Maven - metadata XML file. - - Args: - metadata (str): The XML contents of the Maven metadata file - - Returns: - The latest version of the artifact as a string or None - """ - root = ET.fromstring(metadata) - latest = root.find("./versioning/latest") - if latest is not None: - return latest.text - - return None - - -def _merge_release_please(destination_text: str): - handle_gh_release_key = "handleGHRelease" - branches_key = "branches" - config = yaml.safe_load(destination_text) - if handle_gh_release_key in config: - return destination_text - - config[handle_gh_release_key] = True - - if branches_key in config: - for branch in config[branches_key]: - branch[handle_gh_release_key] = True - return yaml.dump(config) - - -def _merge_common_templates( - source_text: str, destination_text: str, file_path: Path -) -> str: - # keep any existing pom.xml - if file_path.match("pom.xml") or file_path.match("sync-repo-settings.yaml"): - logger.debug(f"existing pom file found ({file_path}) - keeping the existing") - return destination_text - - if file_path.match("release-please.yml"): - return _merge_release_please(destination_text) - - # by default return the newly generated content - return source_text - - -def _common_template_metadata() -> Dict[str, Any]: - metadata = {} # type: Dict[str, Any] - repo_metadata = common._load_repo_metadata() - if repo_metadata: - metadata["repo"] = repo_metadata - group_id, artifact_id = repo_metadata["distribution_name"].split(":") - - metadata["latest_version"] = latest_maven_version( - group_id=group_id, artifact_id=artifact_id - ) - - metadata["latest_bom_version"] = latest_maven_version( - group_id="com.google.cloud", - artifact_id="libraries-bom", - ) - - metadata["samples"] = samples.all_samples(["samples/**/src/main/java/**/*.java"]) - metadata["snippets"] = snippets.all_snippets( - ["samples/**/src/main/java/**/*.java", "samples/**/pom.xml"] - ) - if repo_metadata and "min_java_version" in repo_metadata: - metadata["min_java_version"] = repo_metadata["min_java_version"] - else: - metadata["min_java_version"] = DEFAULT_MIN_SUPPORTED_JAVA_VERSION - - return metadata - - -def common_templates( - excludes: List[str] = [], - template_path: Optional[Path] = None, - **kwargs, -) -> None: - """Generate common templates for a Java Library - - Fetches information about the repository from the .repo-metadata.json file, - information about the latest artifact versions and copies the files into - their expected location. - - Args: - :param excludes: List of template paths to ignore - :param template_path: - :param kwargs: Additional options for CommonTemplates.java_library() - """ - metadata = _common_template_metadata() - kwargs[METADATA] = metadata - - # Generate flat to tell this repository is a split repo that have migrated - # to monorepo. The owlbot.py in the monorepo sets monorepo=True. - monorepo = kwargs.get("monorepo", False) - kwargs["monorepo"] = monorepo - split_repo = not monorepo - repo_metadata = metadata["repo"] - repo_short = repo_metadata["repo_short"] - if os.getenv(LIBRARIES_BOM_VERSION_ENV_KEY, default=None) is not None: - kwargs[METADATA][LIBRARIES_BOM_VERSION] = os.getenv( - LIBRARIES_BOM_VERSION_ENV_KEY - ) - kwargs[METADATA][LIBRARY_VERSION] = os.getenv(LIBRARY_VERSION_ENV_KEY) - # Special libraries that are not GAPIC_AUTO but in the monorepo - special_libs_in_monorepo = [ - "java-translate", - "java-dns", - "java-notification", - "java-resourcemanager", - ] - kwargs["migrated_split_repo"] = split_repo and ( - repo_metadata["library_type"] == "GAPIC_AUTO" - or (repo_short and repo_short in special_libs_in_monorepo) - ) - logger.info( - "monorepo: {}, split_repo: {}, library_type: {}," - " repo_short: {}, migrated_split_repo: {}".format( - monorepo, - split_repo, - repo_metadata["library_type"], - repo_short, - kwargs["migrated_split_repo"], - ) - ) - - templates = gcp.common.CommonTemplates(template_path=template_path).java_library( - **kwargs - ) - - s.copy([templates], excludes=excludes, merge=_merge_common_templates) - - -def custom_templates(files: List[str], **kwargs) -> None: - """Generate custom template files - - Fetches information about the repository from the .repo-metadata.json file, - information about the latest artifact versions and copies the files into - their expected location. - - Args: - files (List[str], optional): List of template paths to include - **kwargs: Additional options for CommonTemplates.render() - """ - kwargs["metadata"] = _common_template_metadata() - kwargs["metadata"]["partials"] = common.load_partials() - for file in files: - template = gcp.CommonTemplates().render(file, **kwargs) - s.copy([template]) - - -def remove_method(filename: str, signature: str): - """Helper to remove an entire method. - - Goes line-by-line to detect the start of the block. Determines - the end of the block by a closing brace at the same indentation - level. This requires the file to be correctly formatted. - - Example: consider the following class: - - class Example { - public void main(String[] args) { - System.out.println("Hello World"); - } - - public String foo() { - return "bar"; - } - } - - To remove the `main` method above, use: - - remove_method('path/to/file', 'public void main(String[] args)') - - Args: - filename (str): Path to source file - signature (str): Full signature of the method to remove. Example: - `public void main(String[] args)`. - """ - lines = [] - leading_regex = None - with open(filename, "r") as fp: - line = fp.readline() - while line: - # for each line, try to find the matching - regex = re.compile("(\\s*)" + re.escape(signature) + ".*") - match = regex.match(line) - if match: - leading_regex = re.compile(match.group(1) + "}") - line = fp.readline() - continue - - # not in a ignore block - preserve the line - if not leading_regex: - lines.append(line) - line = fp.readline() - continue - - # detect the closing tag based on the leading spaces - match = leading_regex.match(line) - if match: - # block is closed, resume capturing content - leading_regex = None - - line = fp.readline() - - with open(filename, "w") as fp: - for line in lines: - # print(line) - fp.write(line) - - -def copy_and_rename_method(filename: str, signature: str, before: str, after: str): - """Helper to make a copy an entire method and rename it. - - Goes line-by-line to detect the start of the block. Determines - the end of the block by a closing brace at the same indentation - level. This requires the file to be correctly formatted. - The method is copied over and renamed in the method signature. - The calls to both methods are separate and unaffected. - - Example: consider the following class: - - class Example { - public void main(String[] args) { - System.out.println("Hello World"); - } - - public String foo() { - return "bar"; - } - } - - To copy and rename the `main` method above, use: - - copy_and_rename_method('path/to/file', 'public void main(String[] args)', - 'main', 'foo1') - - Args: - filename (str): Path to source file - signature (str): Full signature of the method to remove. Example: - `public void main(String[] args)`. - before (str): name of the method to be copied - after (str): new name of the copied method - """ - lines = [] - method = [] - leading_regex = None - with open(filename, "r") as fp: - line = fp.readline() - while line: - # for each line, try to find the matching - regex = re.compile("(\\s*)" + re.escape(signature) + ".*") - match = regex.match(line) - if match: - leading_regex = re.compile(match.group(1) + "}") - lines.append(line) - method.append(line.replace(before, after)) - line = fp.readline() - continue - - lines.append(line) - # not in a ignore block - preserve the line - if leading_regex: - method.append(line) - else: - line = fp.readline() - continue - - # detect the closing tag based on the leading spaces - match = leading_regex.match(line) - if match: - # block is closed, resume capturing content - leading_regex = None - lines.append("\n") - lines.extend(method) - - line = fp.readline() - - with open(filename, "w") as fp: - for line in lines: - # print(line) - fp.write(line) - - -def add_javadoc(filename: str, signature: str, javadoc_type: str, content: List[str]): - """Helper to add a javadoc annoatation to a method. - - Goes line-by-line to detect the start of the block. - Then finds the existing method comment (if it exists). If the - comment already exists, it will append the javadoc annotation - to the javadoc block. Otherwise, it will create a new javadoc - comment block. - - Example: consider the following class: - - class Example { - public void main(String[] args) { - System.out.println("Hello World"); - } - - public String foo() { - return "bar"; - } - } - - To add a javadoc annotation the `main` method above, use: - - add_javadoc('path/to/file', 'public void main(String[] args)', - 'deprecated', 'Please use foo instead.') - - Args: - filename (str): Path to source file - signature (str): Full signature of the method to remove. Example: - `public void main(String[] args)`. - javadoc_type (str): The type of javadoc annotation. Example: `deprecated`. - content (List[str]): The javadoc lines - """ - lines: List[str] = [] - annotations: List[str] = [] - with open(filename, "r") as fp: - line = fp.readline() - while line: - # for each line, try to find the matching - regex = re.compile("(\\s*)" + re.escape(signature) + ".*") - match = regex.match(line) - if match: - leading_spaces = len(line) - len(line.lstrip()) - indent = leading_spaces * " " - last_line = lines.pop() - while last_line.lstrip() and last_line.lstrip()[0] == "@": - annotations.append(last_line) - last_line = lines.pop() - if last_line.strip() == "*/": - first = True - for content_line in content: - if first: - lines.append( - indent - + " * @" - + javadoc_type - + " " - + content_line - + "\n" - ) - first = False - else: - lines.append(indent + " * " + content_line + "\n") - lines.append(last_line) - else: - lines.append(last_line) - lines.append(indent + "/**\n") - first = True - for content_line in content: - if first: - lines.append( - indent - + " * @" - + javadoc_type - + " " - + content_line - + "\n" - ) - first = False - else: - lines.append(indent + " * " + content_line + "\n") - lines.append(indent + " */\n") - lines.extend(annotations[::-1]) - lines.append(line) - line = fp.readline() - - with open(filename, "w") as fp: - for line in lines: - # print(line) - fp.write(line) - - -def annotate_method(filename: str, signature: str, annotation: str): - """Helper to add an annotation to a method. - - Goes line-by-line to detect the start of the block. - Then adds the annotation above the found method signature. - - Example: consider the following class: - - class Example { - public void main(String[] args) { - System.out.println("Hello World"); - } - - public String foo() { - return "bar"; - } - } - - To add an annotation the `main` method above, use: - - annotate_method('path/to/file', 'public void main(String[] args)', - '@Generated()') - - Args: - filename (str): Path to source file - signature (str): Full signature of the method to remove. Example: - `public void main(String[] args)`. - annotation (str): Full annotation. Example: `@Deprecated` - """ - lines: List[str] = [] - with open(filename, "r") as fp: - line = fp.readline() - while line: - # for each line, try to find the matching - regex = re.compile("(\\s*)" + re.escape(signature) + ".*") - match = regex.match(line) - if match: - leading_spaces = len(line) - len(line.lstrip()) - indent = leading_spaces * " " - lines.append(indent + annotation + "\n") - lines.append(line) - line = fp.readline() - - with open(filename, "w") as fp: - for line in lines: - # print(line) - fp.write(line) - - -def deprecate_method(filename: str, signature: str, alternative: str): - """Helper to deprecate a method. - - Goes line-by-line to detect the start of the block. - Then adds the deprecation comment before the method signature. - The @Deprecation annotation is also added. - - Example: consider the following class: - - class Example { - public void main(String[] args) { - System.out.println("Hello World"); - } - - public String foo() { - return "bar"; - } - } - - To deprecate the `main` method above, use: - - deprecate_method('path/to/file', 'public void main(String[] args)', - DEPRECATION_WARNING.format(new_method="foo")) - - Args: - filename (str): Path to source file - signature (str): Full signature of the method to remove. Example: - `public void main(String[] args)`. - alternative: DEPRECATION WARNING: multiline javadoc comment with user - specified leading open/close comment tags - """ - add_javadoc(filename, signature, "deprecated", alternative.splitlines()) - annotate_method(filename, signature, "@Deprecated") diff --git a/library_generation/owlbot/synthtool/sources/templates.py b/library_generation/owlbot/synthtool/sources/templates.py deleted file mode 100644 index 3ba5391f49..0000000000 --- a/library_generation/owlbot/synthtool/sources/templates.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Union, List -from pathlib import Path - -import jinja2 -import tempfile -import re - - -PathOrStr = Union[str, Path] - - -def _make_env(location): - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(str(location)), - autoescape=False, - keep_trailing_newline=True, - ) - return env - - -def _render_to_path(env, template_name, dest, params): - template = env.get_template(template_name) - - output = template.stream(**params) - - if template_name.endswith(".j2"): - template_name = template.name[:-3] - - dest = dest / template_name - dest.parent.mkdir(parents=True, exist_ok=True) - - with dest.open("w") as fh: - output.dump(fh) - - # Copy file mode over - source_path = Path(template.filename) - mode = source_path.stat().st_mode - dest.chmod(mode) - - return dest - - -class Templates: - def __init__(self, location: PathOrStr) -> None: - self.env = _make_env(location) - self.source_path = Path(location) - self.dir = Path(tempfile.mkdtemp()) - - def render(self, template_name: str, subdir: PathOrStr = ".", **kwargs) -> Path: - return _render_to_path(self.env, template_name, self.dir / subdir, kwargs) - - -class TemplateGroup: - def __init__(self, location: PathOrStr, excludes: List[str] = []) -> None: - self.env = _make_env(location) - self.dir = Path(tempfile.mkdtemp()) - self.excludes = excludes - - def render(self, subdir: PathOrStr = ".", **kwargs) -> Path: - for template_name in self.env.list_templates(): - if template_name not in self.excludes: - print(template_name) - _render_to_path(self.env, template_name, self.dir / subdir, kwargs) - else: - print(f"Skipping: {template_name}") - - return self.dir diff --git a/library_generation/owlbot/synthtool/transforms.py b/library_generation/owlbot/synthtool/transforms.py deleted file mode 100644 index cf87995847..0000000000 --- a/library_generation/owlbot/synthtool/transforms.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path -import shutil -from typing import Callable, Iterable, Union, List, Optional -import os -import re -import sys -import logging - -from synthtool import _tracked_paths - -PathOrStr = Union[str, Path] -ListOfPathsOrStrs = Iterable[Union[str, Path]] - -logger = logging.getLogger() -logger.setLevel(logging.DEBUG) - - -class MissingSourceError(Exception): - pass - - -def _expand_paths(paths: ListOfPathsOrStrs, root: PathOrStr = None) -> Iterable[Path]: - """Given a list of globs/paths, expands them into a flat sequence, - expanding globs as necessary.""" - if paths is None: - return [] - - if isinstance(paths, (str, Path)): - paths = [paths] - - if root is None: - root = Path(".") - - # ensure root is a path - root = Path(root) - - # record name of synth script so we don't try to do transforms on it - synth_script_name = sys.argv[0] - - for path in paths: - if isinstance(path, Path): - if path.is_absolute(): - anchor = Path(path.anchor) - remainder = str(path.relative_to(path.anchor)) - yield from anchor.glob(remainder) - else: - yield from root.glob(str(path)) - else: - yield from ( - p - for p in root.glob(path) - if p.absolute() != Path(synth_script_name).absolute() - ) - - -def _filter_files(paths: Iterable[Path]) -> Iterable[Path]: - """Returns only the paths that are files (no directories).""" - - return (path for path in paths if path.is_file() and os.access(path, os.W_OK)) - - -def _merge_file( - source_path: Path, dest_path: Path, merge: Callable[[str, str, Path], str] -): - """ - Writes to the destination the result of merging the source with the - existing destination contents, using the given merge function. - - The merge function must take three arguments: the source contents, the - old destination contents, and a Path to the file to be written. - """ - - with source_path.open("r") as source_file: - source_text = source_file.read() - - with dest_path.open("r+") as dest_file: - dest_text = dest_file.read() - - final_text = merge(source_text, dest_text, dest_path) - - # use the source file's file permission mode - os.chmod(dest_path, os.stat(source_path).st_mode) - if final_text != dest_text: - dest_file.seek(0) - dest_file.write(final_text) - dest_file.truncate() - else: - dest_path.touch() - - -def _copy_dir_to_existing_dir( - source: Path, - destination: Path, - excludes: ListOfPathsOrStrs = None, - merge: Callable[[str, str, Path], str] = None, -) -> bool: - """ - copies files over existing files to an existing directory - this function does not copy empty directories. - - Returns: True if any files were copied, False otherwise. - """ - copied = False - - if not excludes: - excludes = [] - for root, _, files in os.walk(source): - for name in files: - rel_path = str(Path(root).relative_to(source)) - dest_dir = destination / rel_path - dest_path = dest_dir / name - exclude = [ - e - for e in excludes - if ( - Path(e) == _tracked_paths.relativize(root) - or Path(e) == _tracked_paths.relativize(Path(root) / name) - ) - ] - if not exclude: - os.makedirs(str(dest_dir), exist_ok=True) - source_path = Path(os.path.join(root, name)) - if merge is not None and dest_path.is_file(): - try: - _merge_file(source_path, dest_path, merge) - except Exception: - logger.exception( - "_merge_file failed for %s, fall back to copy", - source_path, - ) - shutil.copy2(str(source_path), str(dest_path)) - else: - shutil.copy2(str(source_path), str(dest_path)) - copied = True - - return copied - - -def move( - sources: ListOfPathsOrStrs, - destination: PathOrStr = None, - excludes: ListOfPathsOrStrs = None, - merge: Callable[[str, str, Path], str] = None, - required: bool = False, -) -> bool: - """ - copy file(s) at source to current directory, preserving file mode. - - Args: - sources (ListOfPathsOrStrs): Glob pattern(s) to copy - destination (PathOrStr): Destination folder for copied files - excludes (ListOfPathsOrStrs): Glob pattern(s) of files to skip - merge (Callable[[str, str, Path], str]): Callback function for merging files - if there is an existing file. - required (bool): If required and no source files are copied, throws a MissingSourceError - - Returns: - True if any files were copied, False otherwise. - """ - copied = False - - for source in _expand_paths(sources): - if destination is None: - canonical_destination = _tracked_paths.relativize(source) - else: - canonical_destination = Path(destination) - - if excludes: - excludes = [ - _tracked_paths.relativize(e) for e in _expand_paths(excludes, source) - ] - else: - excludes = [] - if source.is_dir(): - copied = copied or _copy_dir_to_existing_dir( - source, canonical_destination, excludes=excludes, merge=merge - ) - elif source not in excludes: - # copy individual file - if merge is not None and canonical_destination.is_file(): - try: - _merge_file(source, canonical_destination, merge) - except Exception: - logger.exception( - "_merge_file failed for %s, fall back to copy", source - ) - shutil.copy2(source, canonical_destination) - else: - shutil.copy2(source, canonical_destination) - copied = True - - if not copied: - if required: - raise MissingSourceError( - f"No files in sources {sources} were copied. Does the source " - f"contain files?" - ) - else: - logger.warning( - f"No files in sources {sources} were copied. Does the source " - f"contain files?" - ) - - return copied - - -def _replace_in_file(path, expr, replacement): - try: - with path.open("r+") as fh: - return _replace_in_file_handle(fh, expr, replacement) - except UnicodeDecodeError: - pass # It's a binary file. Try again with a binary regular expression. - flags = expr.flags & ~re.UNICODE - expr = re.compile(expr.pattern.encode(), flags) - with path.open("rb+") as fh: - return _replace_in_file_handle(fh, expr, replacement.encode()) - - -def _replace_in_file_handle(fh, expr, replacement): - content = fh.read() - content, count = expr.subn(replacement, content) - - # Don't bother writing the file if we didn't change - # anything. - if count: - fh.seek(0) - fh.write(content) - fh.truncate() - return count - - -def replace( - sources: ListOfPathsOrStrs, before: str, after: str, flags: int = re.MULTILINE -) -> int: - """Replaces occurrences of before with after in all the given sources. - - Returns: - The number of times the text was found and replaced across all files. - """ - expr = re.compile(before, flags=flags or 0) - paths = _filter_files(_expand_paths(sources, ".")) - - if not paths: - logger.warning(f"No files were found in sources {sources} for replace()") - - count_replaced = 0 - for path in paths: - replaced = _replace_in_file(path, expr, after) - count_replaced += replaced - if replaced: - logger.info(f"Replaced {before!r} in {path}.") - - if not count_replaced: - logger.warning( - f"No replacements made in {sources} for pattern {before}, maybe " - "replacement is no longer needed?" - ) - return count_replaced - - -def get_staging_dirs( - default_version: Optional[str] = None, staging_path: Optional[str] = None -) -> List[Path]: - """Returns the list of directories, one per version, copied from - https://github.com/googleapis/googleapis-gen. Will return in lexical sorting - order with the exception of the default_version which will be last (if specified). - - Args: - default_version: the default version of the API. The directory for this version - will be the last item in the returned list if specified. - staging_path: the path to the staging directory. - - Returns: the empty list if no file were copied. - """ - - if staging_path: - staging = Path(staging_path) - else: - staging = Path("owl-bot-staging") - if staging.is_dir(): - # Collect the subdirectories of the staging directory. - versions = [v.name for v in staging.iterdir() if v.is_dir()] - # Reorder the versions so the default version always comes last. - versions = [v for v in versions if v != default_version] - versions.sort() - if default_version is not None: - versions += [default_version] - dirs = [staging / v for v in versions] - for dir in dirs: - _tracked_paths.add(dir) - return dirs - else: - return [] - - -def remove_staging_dirs(): - """Removes all the staging directories.""" - staging = Path("owl-bot-staging") - if staging.is_dir(): - shutil.rmtree(staging) diff --git a/library_generation/owlbot/templates/clirr/clirr-ignored-differences.xml.j2 b/library_generation/owlbot/templates/clirr/clirr-ignored-differences.xml.j2 deleted file mode 100644 index 3fc907802f..0000000000 --- a/library_generation/owlbot/templates/clirr/clirr-ignored-differences.xml.j2 +++ /dev/null @@ -1,80 +0,0 @@ - - - -{% for proto_path in proto_paths %} - 7012 - {{proto_path}}/*OrBuilder - * get*(*) - - - 7012 - {{proto_path}}/*OrBuilder - boolean contains*(*) - - - 7012 - {{proto_path}}/*OrBuilder - boolean has*(*) - - - - 7006 - {{proto_path}}/** - * getDefaultInstanceForType() - ** - - - 7006 - {{proto_path}}/** - * addRepeatedField(*) - ** - - - 7006 - {{proto_path}}/** - * clear() - ** - - - 7006 - {{proto_path}}/** - * clearField(*) - ** - - - 7006 - {{proto_path}}/** - * clearOneof(*) - ** - - - 7006 - {{proto_path}}/** - * clone() - ** - - - 7006 - {{proto_path}}/** - * mergeUnknownFields(*) - ** - - - 7006 - {{proto_path}}/** - * setField(*) - ** - - - 7006 - {{proto_path}}/** - * setRepeatedField(*) - ** - - - 7006 - {{proto_path}}/** - * setUnknownFields(*) - ** - {% endfor %} - diff --git a/library_generation/owlbot/templates/java_library/.github/CODEOWNERS b/library_generation/owlbot/templates/java_library/.github/CODEOWNERS deleted file mode 100644 index 5002a1b08f..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/CODEOWNERS +++ /dev/null @@ -1,20 +0,0 @@ -# Code owners file. -# This file controls who is tagged for review for any given pull request. - -# For syntax help see: -# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax -{% if 'codeowner_team' in metadata['repo'] %} -# The {{ metadata['repo']['codeowner_team'] }} is the default owner for changes in this repo -* @googleapis/yoshi-java {{ metadata['repo']['codeowner_team'] }} -{% if 'library_type' in metadata['repo'] and metadata['repo']['library_type'] != 'GAPIC_AUTO' %} -# for handwritten libraries, keep codeowner_team in .repo-metadata.json as owner -**/*.java {{ metadata['repo']['codeowner_team'] }} -{% endif %} -{% else %} -* @googleapis/yoshi-java -{% endif %} -# The java-samples-reviewers team is the default owner for samples changes -samples/**/*.java @googleapis/java-samples-reviewers - -# Generated snippets should not be owned by samples reviewers -samples/snippets/generated/ @googleapis/yoshi-java diff --git a/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/bug_report.md b/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index c7539a6878..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve - ---- -{% if migrated_split_repo %} -:bus: This library has moved to -[google-cloud-java/{{ metadata['repo']['repo_short'] }}]( -https://github.com/googleapis/google-cloud-java/tree/main/{{ metadata['repo']['repo_short'] }}). -This repository will be archived in the future. -{% endif %} -Thanks for stopping by to let us know something could be better! - -**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. - -Please run down the following list and make sure you've tried the usual "quick fixes": - - - Search the issues already opened: https://github.com/googleapis/{{metadata['repo']['repo_short']}}/issues - - Check for answers on StackOverflow: http://stackoverflow.com/questions/tagged/google-cloud-platform - -If you are still having issues, please include as much information as possible: - -#### Environment details - -1. Specify the API at the beginning of the title. For example, "BigQuery: ..."). - General, Core, and Other are also allowed as types -2. OS type and version: -3. Java version: -4. {{metadata['repo']['name']}} version(s): - -#### Steps to reproduce - - 1. ? - 2. ? - -#### Code example - -```java -// example -``` - -#### Stack trace -``` -Any relevant stacktrace here. -``` - -#### External references such as API reference guides - -- ? - -#### Any additional information below - - -Following these steps guarantees the quickest resolution possible. - -Thanks! diff --git a/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/feature_request.md b/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index f89a7dc59e..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this library - ---- -{% if migrated_split_repo %} -:bus: This library has moved to -[google-cloud-java/{{ metadata['repo']['repo_short'] }}]( -https://github.com/googleapis/google-cloud-java/tree/main/{{ metadata['repo']['repo_short'] }}). -This repository will be archived in the future. -{% endif %} -Thanks for stopping by to let us know something could be better! - -**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. - -**Is your feature request related to a problem? Please describe.** -What the problem is. Example: I'm always frustrated when [...] - -**Describe the solution you'd like** -What you want to happen. - -**Describe alternatives you've considered** -Any alternative solutions or features you've considered. - -**Additional context** -Any other context or screenshots about the feature request. diff --git a/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/support_request.md b/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/support_request.md deleted file mode 100644 index 9958690321..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/support_request.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: Support request -about: If you have a support contract with Google, please create an issue in the Google Cloud Support console. - ---- - -**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. diff --git a/library_generation/owlbot/templates/java_library/.github/PULL_REQUEST_TEMPLATE.md b/library_generation/owlbot/templates/java_library/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index b3640828ab..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,10 +0,0 @@ -Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: -- [ ] Make sure to open an issue as a [bug/issue](https://github.com/{{ metadata['repo']['repo'] }}/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea -- [ ] Ensure the tests and linter pass -- [ ] Code coverage does not decrease (if any source code was changed) -- [ ] Appropriate docs were updated (if necessary) - -Fixes # ☕️ - -If you write sample code, please follow the [samples format]( -https://github.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md). diff --git a/library_generation/owlbot/templates/java_library/.github/auto-label.yaml b/library_generation/owlbot/templates/java_library/.github/auto-label.yaml deleted file mode 100644 index 4caef688b7..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/auto-label.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -requestsize: - enabled: true diff --git a/library_generation/owlbot/templates/java_library/.github/blunderbuss.yml b/library_generation/owlbot/templates/java_library/.github/blunderbuss.yml deleted file mode 100644 index 2176b05432..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/blunderbuss.yml +++ /dev/null @@ -1,7 +0,0 @@ -# Configuration for the Blunderbuss GitHub app. For more info see -# https://github.com/googleapis/repo-automation-bots/tree/main/packages/blunderbuss -assign_prs_by: -- labels: - - samples - to: - - googleapis/java-samples-reviewers \ No newline at end of file diff --git a/library_generation/owlbot/templates/java_library/.github/dependabot.yml b/library_generation/owlbot/templates/java_library/.github/dependabot.yml deleted file mode 100644 index 203f9eaccf..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/dependabot.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "maven" - directory: "/" - schedule: - interval: "daily" - # Disable version updates for Maven dependencies - # we use renovate-bot as well as shared-dependencies BOM to update maven dependencies. - ignore: - - dependency-name: "*" - - package-ecosystem: "pip" - directory: "/" - schedule: - interval: "daily" - # Disable version updates for pip dependencies - # If a security vulnerability comes in, we will be notified about - # it via template in the synthtool repository. - ignore: - - dependency-name: "*" diff --git a/library_generation/owlbot/templates/java_library/.github/release-please.yml b/library_generation/owlbot/templates/java_library/.github/release-please.yml deleted file mode 100644 index 8ca7f9cabc..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/release-please.yml +++ /dev/null @@ -1,3 +0,0 @@ -bumpMinorPreMajor: true -handleGHRelease: true -releaseType: java-yoshi diff --git a/library_generation/owlbot/templates/java_library/.github/release-trigger.yml b/library_generation/owlbot/templates/java_library/.github/release-trigger.yml deleted file mode 100644 index 5056d3a13b..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/release-trigger.yml +++ /dev/null @@ -1,2 +0,0 @@ -enabled: true -multiScmName: {{ metadata['repo']['repo_short'] }} diff --git a/library_generation/owlbot/templates/java_library/.github/scripts/update_generation_config.sh b/library_generation/owlbot/templates/java_library/.github/scripts/update_generation_config.sh deleted file mode 100644 index 561a313040..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/scripts/update_generation_config.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/bash -set -e -# This script should be run at the root of the repository. -# This script is used to update googleapis_commitish, gapic_generator_version, -# and libraries_bom_version in generation configuration at the time of running -# and create a pull request. - -# The following commands need to be installed before running the script: -# 1. git -# 2. gh -# 3. jq - -# Utility functions -# Get the latest released version of a Maven artifact. -function get_latest_released_version() { - local group_id=$1 - local artifact_id=$2 - latest=$(curl -s "https://search.maven.org/solrsearch/select?q=g:${group_id}+AND+a:${artifact_id}&core=gav&rows=500&wt=json" | jq -r '.response.docs[] | select(.v | test("^[0-9]+(\\.[0-9]+)*$")) | .v' | sort -V | tail -n 1) - echo "${latest}" -} - -# Update a key to a new value in the generation config. -function update_config() { - local key_word=$1 - local new_value=$2 - local file=$3 - echo "Update ${key_word} to ${new_value} in ${file}" - sed -i -e "s/^${key_word}.*$/${key_word}: ${new_value}/" "${file}" -} - -# The parameters of this script is: -# 1. base_branch, the base branch of the result pull request. -# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java -# 3. [optional] generation_config, the path to the generation configuration, -# the default value is generation_config.yaml in the repository root. -while [[ $# -gt 0 ]]; do -key="$1" -case "${key}" in - --base_branch) - base_branch="$2" - shift - ;; - --repo) - repo="$2" - shift - ;; - --generation_config) - generation_config="$2" - shift - ;; - *) - echo "Invalid option: [$1]" - exit 1 - ;; -esac -shift -done - -if [ -z "${base_branch}" ]; then - echo "missing required argument --base_branch" - exit 1 -fi - -if [ -z "${repo}" ]; then - echo "missing required argument --repo" - exit 1 -fi - -if [ -z "${generation_config}" ]; then - generation_config="generation_config.yaml" - echo "Use default generation config: ${generation_config}" -fi - -current_branch="generate-libraries-${base_branch}" -title="chore: Update generation configuration at $(date)" - -# try to find a open pull request associated with the branch -pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") -# create a branch if there's no open pull request associated with the -# branch; otherwise checkout the pull request. -if [ -z "${pr_num}" ]; then - git checkout -b "${current_branch}" -else - gh pr checkout "${pr_num}" -fi - -mkdir tmp-googleapis -# use partial clone because only commit history is needed. -git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis -pushd tmp-googleapis -git pull -latest_commit=$(git rev-parse HEAD) -popd -rm -rf tmp-googleapis -update_config "googleapis_commitish" "${latest_commit}" "${generation_config}" - -# update gapic-generator-java version to the latest -latest_version=$(get_latest_released_version "com.google.api" "gapic-generator-java") -update_config "gapic_generator_version" "${latest_version}" "${generation_config}" - -# update libraries-bom version to the latest -latest_version=$(get_latest_released_version "com.google.cloud" "libraries-bom") -update_config "libraries_bom_version" "${latest_version}" "${generation_config}" - -git add "${generation_config}" -changed_files=$(git diff --cached --name-only) -if [[ "${changed_files}" == "" ]]; then - echo "The latest generation config is not changed." - echo "Skip committing to the pull request." - exit 0 -fi -git commit -m "${title}" -if [ -z "${pr_num}" ]; then - git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" - git fetch -q --unshallow remote_repo - git push -f remote_repo "${current_branch}" - gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" -else - git push - gh pr edit "${pr_num}" --title "${title}" --body "${title}" -fi diff --git a/library_generation/owlbot/templates/java_library/.github/snippet-bot.yml b/library_generation/owlbot/templates/java_library/.github/snippet-bot.yml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml b/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml deleted file mode 100644 index bbfd4c0314..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Whether or not rebase-merging is enabled on this repository. -# Defaults to `true` -rebaseMergeAllowed: false - -# Whether or not squash-merging is enabled on this repository. -# Defaults to `true` -squashMergeAllowed: true - -# Whether or not PRs are merged with a merge commit on this repository. -# Defaults to `false` -mergeCommitAllowed: false - -# Rules for main branch protection -branchProtectionRules: -# Identifies the protection rule pattern. Name of the branch to be protected. -# Defaults to `main` -- pattern: main - # Can admins overwrite branch protection. - # Defaults to `true` - isAdminEnforced: true - # Number of approving reviews required to update matching branches. - # Defaults to `1` - requiredApprovingReviewCount: 1 - # Are reviews from code owners required to update matching branches. - # Defaults to `false` - requiresCodeOwnerReviews: true - # Require up to date branches - requiresStrictStatusChecks: false - # List of required status check contexts that must pass for commits to be accepted to matching branches. - requiredStatusCheckContexts: - - "dependencies (17)" - - "lint" - - "javadoc" - - "units (8)" - - "units (11)" - - "Kokoro - Test: Integration" - - "cla/google" - - "OwlBot Post Processor" - - "Kokoro - Test: Java GraalVM Native Image" - - "Kokoro - Test: Java 17 GraalVM Native Image" -# List of explicit permissions to add (additive only) -permissionRules: -- team: yoshi-admins - permission: admin -- team: yoshi-java-admins - permission: admin -- team: yoshi-java - permission: push -- team: java-samples-reviewers - permission: push - diff --git a/library_generation/owlbot/templates/java_library/.github/trusted-contribution.yml b/library_generation/owlbot/templates/java_library/.github/trusted-contribution.yml deleted file mode 100644 index 88d3ac9bf1..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/trusted-contribution.yml +++ /dev/null @@ -1,9 +0,0 @@ -trustedContributors: -- renovate-bot -- gcf-owl-bot[bot] - -annotations: -- type: comment - text: "/gcbrun" -- type: label - text: "kokoro:force-run" diff --git a/library_generation/owlbot/templates/java_library/.github/workflows/approve-readme.yaml b/library_generation/owlbot/templates/java_library/.github/workflows/approve-readme.yaml deleted file mode 100644 index bbef6d37cc..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/workflows/approve-readme.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Github action job to test core java library features on -# downstream client libraries before they are released. -on: - pull_request: -name: auto-merge-readme -jobs: - approve: - runs-on: ubuntu-latest - if: github.repository_owner == 'googleapis' && github.head_ref == 'autosynth-readme' - steps: - - uses: actions/github-script@v7 - with: - github-token: {{ '${{secrets.YOSHI_APPROVER_TOKEN}}' }} - script: | - // only approve PRs from yoshi-automation - if (context.payload.pull_request.user.login !== "yoshi-automation") { - return; - } - - // only approve PRs like "chore: release " - if (!context.payload.pull_request.title === "chore: regenerate README") { - return; - } - - // only approve PRs with README.md and synth.metadata changes - const files = new Set( - ( - await github.paginate( - github.pulls.listFiles.endpoint({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number, - }) - ) - ).map(file => file.filename) - ); - if (files.size != 2 || !files.has("README.md") || !files.has(".github/readme/synth.metadata/synth.metadata")) { - return; - } - - // approve README regeneration PR - await github.pulls.createReview({ - owner: context.repo.owner, - repo: context.repo.repo, - body: 'Rubber stamped PR!', - pull_number: context.payload.pull_request.number, - event: 'APPROVE' - }); - - // attach automerge label - await github.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - labels: ['automerge'] - }); diff --git a/library_generation/owlbot/templates/java_library/.github/workflows/ci.yaml b/library_generation/owlbot/templates/java_library/.github/workflows/ci.yaml deleted file mode 100644 index 50487eeb3b..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/workflows/ci.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Github action job to test core java library features on -# downstream client libraries before they are released. -on: - push: - branches: - - main - pull_request: -name: ci -jobs: - units: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - java: [11, 17, 21] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: ${{'{{matrix.java}}'}} - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: test - units-java8: - # Building using Java 17 and run the tests with Java 8 runtime - name: "units (8)" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - java-version: 8 - distribution: temurin - - name: "Set jvm system property environment variable for surefire plugin (unit tests)" - # Maven surefire plugin (unit tests) allows us to specify JVM to run the tests. - # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm - run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java" >> $GITHUB_ENV - shell: bash - - uses: actions/setup-java@v4 - with: - java-version: 17 - distribution: temurin - - run: .kokoro/build.sh - env: - JOB_TYPE: test - windows: - runs-on: windows-latest - steps: - - name: Support longpaths - run: git config --system core.longpaths true - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 8 - - run: java -version - - run: .kokoro/build.bat - env: - JOB_TYPE: test - dependencies: - runs-on: ubuntu-latest - strategy: - matrix: - java: [17] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: ${{'{{matrix.java}}'}} - - run: java -version - - run: .kokoro/dependencies.sh - javadoc: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 17 - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: javadoc - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 11 - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: lint - clirr: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 8 - - run: java -version - - run: .kokoro/build.sh - env: - JOB_TYPE: clirr diff --git a/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml b/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml deleted file mode 100644 index 7c5ec7865e..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Renovate Bot Config Validation - -on: - pull_request: - paths: - - 'renovate.json' - -jobs: - renovate_bot_config_validation: - runs-on: ubuntu-22.04 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Install Renovate and Config Validator - run: | - npm install -g npm@latest - npm install --global renovate - renovate-config-validator diff --git a/library_generation/owlbot/templates/java_library/.github/workflows/samples.yaml b/library_generation/owlbot/templates/java_library/.github/workflows/samples.yaml deleted file mode 100644 index 03b2939567..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/workflows/samples.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Github action job to test core java library features on -# downstream client libraries before they are released. -on: - pull_request: -name: samples -jobs: - checkstyle: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 8 - - name: Run checkstyle - run: mvn -P lint --quiet --batch-mode checkstyle:check - working-directory: samples/snippets diff --git a/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml b/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml deleted file mode 100644 index 0dbdc19a4a..0000000000 --- a/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# GitHub action job to test core java library features on -# downstream client libraries before they are released. -name: Update generation configuration -on: - schedule: - - cron: '0 2 * * *' - workflow_dispatch: -{% raw %} -jobs: - update-generation-config: - runs-on: ubuntu-22.04 - env: - # the branch into which the pull request is merged - base_branch: main - steps: - - uses: actions/checkout@v4 - with: - token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} - - name: Update params in generation config to latest - shell: bash - run: | - set -x - [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" - [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" - bash .github/scripts/update_generation_config.sh \ - --base_branch "${base_branch}" \ - --repo ${{ github.repository }} - env: - GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} -{% endraw %} diff --git a/library_generation/owlbot/templates/java_library/.kokoro/build.sh b/library_generation/owlbot/templates/java_library/.kokoro/build.sh deleted file mode 100755 index eda70322e3..0000000000 --- a/library_generation/owlbot/templates/java_library/.kokoro/build.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -cd github/synthtool - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Run tests -nox -s lint test - -# remove all files, preventing kokoro from trying to sync them. -rm -rf * diff --git a/library_generation/owlbot/templates/java_library/.kokoro/common.cfg b/library_generation/owlbot/templates/java_library/.kokoro/common.cfg deleted file mode 100644 index 653d0e51ec..0000000000 --- a/library_generation/owlbot/templates/java_library/.kokoro/common.cfg +++ /dev/null @@ -1,19 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "synthtool/.kokoro/trampoline.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python" -} - -# Tell the trampoline which build file to use. -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/synthtool/.kokoro/build.sh" -} diff --git a/library_generation/owlbot/templates/java_library/.kokoro/continuous.cfg b/library_generation/owlbot/templates/java_library/.kokoro/continuous.cfg deleted file mode 100644 index 18a4c35325..0000000000 --- a/library_generation/owlbot/templates/java_library/.kokoro/continuous.cfg +++ /dev/null @@ -1 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto diff --git a/library_generation/owlbot/templates/java_library/.kokoro/presubmit.cfg b/library_generation/owlbot/templates/java_library/.kokoro/presubmit.cfg deleted file mode 100644 index 18a4c35325..0000000000 --- a/library_generation/owlbot/templates/java_library/.kokoro/presubmit.cfg +++ /dev/null @@ -1 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto diff --git a/library_generation/owlbot/templates/java_library/.kokoro/trampoline.sh b/library_generation/owlbot/templates/java_library/.kokoro/trampoline.sh deleted file mode 100755 index 0efc3be388..0000000000 --- a/library_generation/owlbot/templates/java_library/.kokoro/trampoline.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -eo pipefail -# Always run the cleanup script, regardless of the success of bouncing into -# the container. -function cleanup() { - chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh - ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh - echo "cleanup"; -} -trap cleanup EXIT -python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" diff --git a/library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md b/library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md deleted file mode 100644 index 2add2547a8..0000000000 --- a/library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,94 +0,0 @@ - -# Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of -experience, education, socio-economic status, nationality, personal appearance, -race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, or to ban temporarily or permanently any -contributor for other behaviors that they deem inappropriate, threatening, -offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -This Code of Conduct also applies outside the project spaces when the Project -Steward has a reasonable belief that an individual's behavior may have a -negative impact on the project or its community. - -## Conflict Resolution - -We do not believe that all conflict is bad; healthy debate and disagreement -often yield positive results. However, it is never okay to be disrespectful or -to engage in behavior that violates the project’s code of conduct. - -If you see someone violating the code of conduct, you are encouraged to address -the behavior directly with those involved. Many issues can be resolved quickly -and easily, and this gives people more control over the outcome of their -dispute. If you are unable to resolve the matter for any reason, or if the -behavior is threatening or harassing, report it. We are dedicated to providing -an environment where participants feel welcome and safe. - -Reports should be directed to *googleapis-stewards@google.com*, the -Project Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to -receive and address reported violations of the code of conduct. They will then -work with a committee consisting of representatives from the Open Source -Programs Office and the Google Open Source Strategy team. If for any reason you -are uncomfortable reaching out to the Project Steward, please email -opensource@google.com. - -We will investigate every complaint, but you may not receive a direct response. -We will use our discretion in determining when and how to follow up on reported -incidents, which may range from not taking action to permanent expulsion from -the project and project-sponsored spaces. We will notify the accused of the -report and provide them an opportunity to discuss it before any action is taken. -The identity of the reporter will be omitted from the details of the report -supplied to the accused. In potentially harmful situations, such as ongoing -harassment or threats to anyone's safety, we may take action without notice. - -## Attribution - -This Code of Conduct is adapted from the Contributor Covenant, version 1.4, -available at -https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file diff --git a/library_generation/owlbot/templates/java_library/CONTRIBUTING.md b/library_generation/owlbot/templates/java_library/CONTRIBUTING.md deleted file mode 100644 index b65dd279c9..0000000000 --- a/library_generation/owlbot/templates/java_library/CONTRIBUTING.md +++ /dev/null @@ -1,92 +0,0 @@ -# How to Contribute - -We'd love to accept your patches and contributions to this project. There are -just a few small guidelines you need to follow. - -## Contributor License Agreement - -Contributions to this project must be accompanied by a Contributor License -Agreement. You (or your employer) retain the copyright to your contribution; -this simply gives us permission to use and redistribute your contributions as -part of the project. Head over to to see -your current agreements on file or to sign a new one. - -You generally only need to submit a CLA once, so if you've already submitted one -(even if it was for a different project), you probably don't need to do it -again. - -## Code reviews - -All submissions, including submissions by project members, require review. We -use GitHub pull requests for this purpose. Consult -[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more -information on using pull requests. - -## Community Guidelines - -This project follows -[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). - -## Building the project - -To build, package, and run all unit tests run the command - -``` -mvn clean verify -``` - -### Running Integration tests - -To include integration tests when building the project, you need access to -a GCP Project with a valid service account. - -For instructions on how to generate a service account and corresponding -credentials JSON see: [Creating a Service Account][1]. - -Then run the following to build, package, run all unit tests and run all -integration tests. - -```bash -export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account.json -mvn -Penable-integration-tests clean verify -``` - -## Code Samples - -All code samples must be in compliance with the [java sample formatting guide][3]. -Code Samples must be bundled in separate Maven modules. - -The samples must be separate from the primary project for a few reasons: -1. Primary projects have a minimum Java version of Java 8 whereas samples can have - Java version of Java 11. Due to this we need the ability to - selectively exclude samples from a build run. -2. Many code samples depend on external GCP services and need - credentials to access the service. -3. Code samples are not released as Maven artifacts and must be excluded from - release builds. - -### Building - -```bash -mvn clean verify -``` - -Some samples require access to GCP services and require a service account: - -```bash -export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account.json -mvn clean verify -``` - -### Code Formatting - -Code in this repo is formatted with -[google-java-format](https://github.com/google/google-java-format). -To run formatting on your project, you can run: -``` -mvn com.coveo:fmt-maven-plugin:format -``` - -[1]: https://cloud.google.com/docs/authentication/getting-started#creating_a_service_account -[2]: https://maven.apache.org/settings.html#Active_Profiles -[3]: https://github.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md \ No newline at end of file diff --git a/library_generation/owlbot/templates/java_library/LICENSE b/library_generation/owlbot/templates/java_library/LICENSE deleted file mode 100644 index 261eeb9e9f..0000000000 --- a/library_generation/owlbot/templates/java_library/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md deleted file mode 100644 index 35b1b34492..0000000000 --- a/library_generation/owlbot/templates/java_library/README.md +++ /dev/null @@ -1,288 +0,0 @@ -{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} -{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} -{% set version = metadata['library_version'] -%} -{% set repo_short = metadata['repo']['repo'].split('/')|last -%} - -# Google {{ metadata['repo']['name_pretty'] }} Client for Java - -Java idiomatic client for [{{metadata['repo']['name_pretty']}}][product-docs]. - -[![Maven][maven-version-image]][maven-version-link] -![Stability][stability-image] - -- [Product Documentation][product-docs] -- [Client Library Documentation][javadocs] -{% if 'partials' in metadata and metadata['partials']['deprecation_warning'] -%} -{{ metadata['partials']['deprecation_warning'] }} -{% elif metadata['repo']['release_level'] in ['preview'] %} -> Note: This client is a work-in-progress, and may occasionally -> make backwards-incompatible changes. -{% endif %} -{% if migrated_split_repo %} -:bus: In October 2022, this library has moved to -[google-cloud-java/{{ metadata['repo']['repo_short'] }}]( -https://github.com/googleapis/google-cloud-java/tree/main/{{ metadata['repo']['repo_short'] }}). -This repository will be archived in the future. -Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). -The Maven artifact coordinates (`{{ group_id }}:{{ artifact_id }}`) remain the same. -{% endif %} -## Quickstart - -{% if 'snippets' in metadata and metadata['snippets'][metadata['repo']['api_shortname'] + '_install_with_bom'] -%} -If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: - -```xml -{{ metadata['snippets'][metadata['repo']['api_shortname'] + '_install_with_bom'] }} -``` - -If you are using Maven without the BOM, add this to your dependencies: -{% elif monorepo %} -If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: - -```xml - - - - com.google.cloud - libraries-bom - {{ metadata['libraries_bom_version'] }} - pom - import - - - - - - - {{ group_id }} - {{ artifact_id }} - - -``` - -If you are using Maven without the BOM, add this to your dependencies: -{% else %} -If you are using Maven, add this to your pom.xml file: -{% endif %} - -```xml -{% if 'snippets' in metadata and metadata['snippets'][metadata['repo']['api_shortname'] + '_install_without_bom'] -%} -{{ metadata['snippets'][metadata['repo']['api_shortname'] + '_install_without_bom'] }} -{% else -%} - - {{ group_id }} - {{ artifact_id }} - {{ version }} - -{% endif -%} -``` - -{% if 'snippets' in metadata and metadata['snippets'][metadata['repo']['api_shortname'] + '_install_with_bom'] -%} -If you are using Gradle 5.x or later, add this to your dependencies: - -```Groovy -implementation platform('com.google.cloud:libraries-bom:{{metadata['libraries_bom_version']}}') - -implementation '{{ group_id }}:{{ artifact_id }}' -``` -{% endif -%} - -If you are using Gradle without BOM, add this to your dependencies: - -```Groovy -implementation '{{ group_id }}:{{ artifact_id }}:{{ version }}' -``` - -If you are using SBT, add this to your dependencies: - -```Scala -libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ version }}" -``` - -## Authentication - -See the [Authentication][authentication] section in the base directory's README. - -## Authorization - -The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired {{metadata['repo']['name_pretty']}} APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the {{metadata['repo']['name_pretty']}} API calls. - -## Getting Started - -### Prerequisites - -You will need a [Google Cloud Platform Console][developer-console] project with the {{metadata['repo']['name_pretty']}} [API enabled][enable-api]. -{% if metadata['repo']['requires_billing'] %}You will need to [enable billing][enable-billing] to use Google {{metadata['repo']['name_pretty']}}.{% endif %} -[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by -[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: -`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. - -### Installation and setup - -You'll need to obtain the `{{ artifact_id }}` library. See the [Quickstart](#quickstart) section -to add `{{ artifact_id }}` as a dependency in your code. - -## About {{metadata['repo']['name_pretty']}} - -{% if 'partials' in metadata and metadata['partials']['about'] -%} -{{ metadata['partials']['about'] }} -{% else %} -[{{ metadata['repo']['name_pretty'] }}][product-docs] {{ metadata['repo']['api_description'] }} - -See the [{{metadata['repo']['name_pretty']}} client library docs][javadocs] to learn how to -use this {{metadata['repo']['name_pretty']}} Client Library. -{% endif %} - -{% if 'partials' in metadata and metadata['partials']['custom_content'] -%} -{{ metadata['partials']['custom_content'] }} -{% endif %} - -{% if metadata['samples']|length %} -## Samples - -Samples are in the [`samples/`](https://github.com/{{ metadata['repo']['repo'] }}/tree/main/samples) directory. - -| Sample | Source Code | Try it | -| --------------------------- | --------------------------------- | ------ | -{% for sample in metadata['samples'] %}| {{ sample.title }} | [source code](https://github.com/{{ metadata['repo']['repo'] }}/blob/main/{{ sample.file }}) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/{{ metadata['repo']['repo'] }}&page=editor&open_in_editor={{ sample.file }}) | -{% endfor %} -{% endif %} - -## Troubleshooting - -To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. - -{% if metadata['repo']['transport'] -%} -## Transport - -{% if metadata['repo']['transport'] == 'grpc' -%} -{{metadata['repo']['name_pretty']}} uses gRPC for the transport layer. -{% elif metadata['repo']['transport'] == 'http' -%} -{{metadata['repo']['name_pretty']}} uses HTTP/JSON for the transport layer. -{% elif metadata['repo']['transport'] == 'both' -%} -{{metadata['repo']['name_pretty']}} uses both gRPC and HTTP/JSON for the transport layer. -{% endif %} -{% endif -%} - -## Supported Java Versions - -Java {{ metadata['min_java_version'] }} or above is required for using this client. - -Google's Java client libraries, -[Google Cloud Client Libraries][cloudlibs] -and -[Google Cloud API Libraries][apilibs], -follow the -[Oracle Java SE support roadmap][oracle] -(see the Oracle Java SE Product Releases section). - -### For new development - -In general, new feature development occurs with support for the lowest Java -LTS version covered by Oracle's Premier Support (which typically lasts 5 years -from initial General Availability). If the minimum required JVM for a given -library is changed, it is accompanied by a [semver][semver] major release. - -Java 11 and (in September 2021) Java 17 are the best choices for new -development. - -### Keeping production systems current - -Google tests its client libraries with all current LTS versions covered by -Oracle's Extended Support (which typically lasts 8 years from initial -General Availability). - -#### Legacy support - -Google's client libraries support legacy versions of Java runtimes with long -term stable libraries that don't receive feature updates on a best efforts basis -as it may not be possible to backport all patches. - -Google provides updates on a best efforts basis to apps that continue to use -Java 7, though apps might need to upgrade to current versions of the library -that supports their JVM. - -#### Where to find specific information - -The latest versions and the supported Java versions are identified on -the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` -and on [google-cloud-java][g-c-j]. - -## Versioning - -{% if 'partials' in metadata and metadata['partials']['versioning'] -%} -{{ metadata['partials']['versioning'] }} -{% else %} -This library follows [Semantic Versioning](http://semver.org/). - -{% if metadata['repo']['release_level'] in ['preview'] %} -It is currently in major version zero (``0.y.z``), which means that anything may change at any time -and the public API should not be considered stable. -{% endif %}{% endif %} - -## Contributing - -{% if 'partials' in metadata and metadata['partials']['contributing'] -%} -{{ metadata['partials']['contributing'] }} -{% else %} -Contributions to this library are always welcome and highly encouraged. - -See [CONTRIBUTING][contributing] for more information how to get started. - -Please note that this project is released with a Contributor Code of Conduct. By participating in -this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more -information. -{% endif %} - -## License - -Apache 2.0 - See [LICENSE][license] for more information. - -## CI Status - -Java Version | Status ------------- | ------{% if metadata['min_java_version'] <= 7 %} -Java 7 | [![Kokoro CI][kokoro-badge-image-1]][kokoro-badge-link-1]{% endif %} -Java 8 | [![Kokoro CI][kokoro-badge-image-2]][kokoro-badge-link-2] -Java 8 OSX | [![Kokoro CI][kokoro-badge-image-3]][kokoro-badge-link-3] -Java 8 Windows | [![Kokoro CI][kokoro-badge-image-4]][kokoro-badge-link-4] -Java 11 | [![Kokoro CI][kokoro-badge-image-5]][kokoro-badge-link-5] - -Java is a registered trademark of Oracle and/or its affiliates. - -[product-docs]: {{metadata['repo']['product_documentation']}} -[javadocs]: {{metadata['repo']['client_documentation']}} -[kokoro-badge-image-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java7.svg -[kokoro-badge-link-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java7.html -[kokoro-badge-image-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8.svg -[kokoro-badge-link-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8.html -[kokoro-badge-image-3]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-osx.svg -[kokoro-badge-link-3]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-osx.html -[kokoro-badge-image-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-win.svg -[kokoro-badge-link-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-win.html -[kokoro-badge-image-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.svg -[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.html -[stability-image]: https://img.shields.io/badge/stability-{% if metadata['repo']['release_level'] == 'stable' %}stable-green{% elif metadata['repo']['release_level'] == 'preview' %}preview-yellow{% else %}unknown-red{% endif %} -[maven-version-image]: https://img.shields.io/maven-central/v/{{ group_id }}/{{ artifact_id }}.svg -[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ version }} -[authentication]: https://github.com/googleapis/google-cloud-java#authentication -[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes -[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles -[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy -[developer-console]: https://console.developers.google.com/ -[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects -[cloud-cli]: https://cloud.google.com/cli -[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md -[contributing]: https://github.com/{{metadata['repo']['repo']}}/blob/main/CONTRIBUTING.md -[code-of-conduct]: https://github.com/{{metadata['repo']['repo']}}/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct -[license]: https://github.com/{{metadata['repo']['repo']}}/blob/main/LICENSE -{% if metadata['repo']['requires_billing'] %}[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing{% endif %} -{% if metadata['repo']['api_id'] %}[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid={{ metadata['repo']['api_id'] }}{% endif %} -[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM -[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png - -[semver]: https://semver.org/ -[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained -[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries -[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html -[g-c-j]: http://github.com/googleapis/google-cloud-java diff --git a/library_generation/owlbot/templates/java_library/SECURITY.md b/library_generation/owlbot/templates/java_library/SECURITY.md deleted file mode 100644 index 8b58ae9c01..0000000000 --- a/library_generation/owlbot/templates/java_library/SECURITY.md +++ /dev/null @@ -1,7 +0,0 @@ -# Security Policy - -To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). - -The Google Security Team will respond within 5 working days of your report on g.co/vulnz. - -We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. diff --git a/library_generation/owlbot/templates/java_library/java.header b/library_generation/owlbot/templates/java_library/java.header deleted file mode 100644 index d0970ba7d3..0000000000 --- a/library_generation/owlbot/templates/java_library/java.header +++ /dev/null @@ -1,15 +0,0 @@ -^/\*$ -^ \* Copyright \d\d\d\d,? Google (Inc\.|LLC)$ -^ \*$ -^ \* Licensed under the Apache License, Version 2\.0 \(the "License"\);$ -^ \* you may not use this file except in compliance with the License\.$ -^ \* You may obtain a copy of the License at$ -^ \*$ -^ \*[ ]+https?://www.apache.org/licenses/LICENSE-2\.0$ -^ \*$ -^ \* Unless required by applicable law or agreed to in writing, software$ -^ \* distributed under the License is distributed on an "AS IS" BASIS,$ -^ \* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied\.$ -^ \* See the License for the specific language governing permissions and$ -^ \* limitations under the License\.$ -^ \*/$ diff --git a/library_generation/owlbot/templates/java_library/license-checks.xml b/library_generation/owlbot/templates/java_library/license-checks.xml deleted file mode 100644 index 6597fced80..0000000000 --- a/library_generation/owlbot/templates/java_library/license-checks.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - diff --git a/library_generation/owlbot/templates/java_library/renovate.json b/library_generation/owlbot/templates/java_library/renovate.json deleted file mode 100644 index cb92a78173..0000000000 --- a/library_generation/owlbot/templates/java_library/renovate.json +++ /dev/null @@ -1,128 +0,0 @@ -{% if migrated_split_repo %} -{ - "enabled": false, -{% else %} -{ -{% endif %} - "extends": [ - ":separateMajorReleases", - ":combinePatchMinorReleases", - ":ignoreUnstable", - ":prImmediately", - ":updateNotScheduled", - ":automergeDisabled", - ":ignoreModulesAndTests", - ":maintainLockFilesDisabled", - ":autodetectPinVersions" - ], - "ignorePaths": [ - ".kokoro/requirements.txt", - ".github/workflows/approve-readme.yaml", - ".github/workflows/ci.yaml", - ".github/workflows/renovate_config_check.yaml", - ".github/workflows/samples.yaml" - ], - "customManagers": [ - { - "customType": "regex", - "fileMatch": [ - "^.kokoro/presubmit/graalvm-native.*.cfg$" - ], - "matchStrings": [ - "value: \"gcr.io/cloud-devrel-public-resources/graalvm.*:(?.*?)\"" - ], - "depNameTemplate": "com.google.cloud:sdk-platform-java-config", - "datasourceTemplate": "maven" - }, - { - "customType": "regex", - "fileMatch": [ - "^.github/workflows/unmanaged_dependency_check.yaml$" - ], - "matchStrings": [ - "uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v(?.+?)\\n" - ], - "depNameTemplate": "com.google.cloud:sdk-platform-java-config", - "datasourceTemplate": "maven" - }, - { - "fileMatch": [ - "^.github/workflows/hermetic_library_generation.yaml$" - ], - "matchStrings": [ - "uses: googleapis/sdk-platform-java/.github/scripts@v(?.+?)\\n" - ], - "depNameTemplate": "com.google.api:gapic-generator-java", - "datasourceTemplate": "maven" - } - ], - "packageRules": [ - { - "packagePatterns": [ - "^com.google.guava:" - ], - "versionScheme": "docker" - }, - { - "packagePatterns": [ - "*" - ], - "semanticCommitType": "deps", - "semanticCommitScope": null - }, - { - "packagePatterns": [ - "^org.apache.maven", - "^org.jacoco:", - "^org.codehaus.mojo:", - "^org.sonatype.plugins:", - "^com.coveo:", - "^com.google.cloud:google-cloud-shared-config" - ], - "semanticCommitType": "build", - "semanticCommitScope": "deps" - }, - { - "packagePatterns": [ - "^{{metadata['repo']['distribution_name']}}", - "^com.google.cloud:libraries-bom", - "^com.google.cloud.samples:shared-configuration" - ], - "semanticCommitType": "chore", - "semanticCommitScope": "deps" - }, - { - "packagePatterns": [ - "^junit:junit", - "^com.google.truth:truth", - "^org.mockito:mockito-core", - "^org.objenesis:objenesis", - "^com.google.cloud:google-cloud-conformance-tests", - "^org.graalvm.buildtools:junit-platform-native" - ], - "semanticCommitType": "test", - "semanticCommitScope": "deps" - }, - { - "packagePatterns": [ - "^com.google.cloud:google-cloud-" - ], - "ignoreUnstable": false - }, - { - "packagePatterns": [ - "^com.fasterxml.jackson.core" - ], - "groupName": "jackson dependencies" - }, - { - "matchPackagePatterns": [ - "^com.google.api:gapic-generator-java", - "^com.google.cloud:sdk-platform-java-config" - ], - "groupName": "SDK platform Java dependencies" - } - ], - "semanticCommits": true, - "dependencyDashboard": true -} \ No newline at end of file diff --git a/library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml b/library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml deleted file mode 100644 index 110250d003..0000000000 --- a/library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ -{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} -{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} - - - 4.0.0 - com.google.cloud - {{metadata['repo']['name']}}-install-without-bom - jar - Google {{metadata['repo']['name_pretty']}} Install Without Bom - https://github.com/{{metadata['repo']['repo']}} - - - - com.google.cloud.samples - shared-configuration - 1.2.0 - - - - 1.8 - 1.8 - UTF-8 - - - - - - - {{ group_id }} - {{ artifact_id }} - {{ metadata['latest_version'] }} - - - - - junit - junit - 4.13.2 - test - - - com.google.truth - truth - 1.1.3 - test - - - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.3.0 - - - add-snippets-source - - add-source - - - - ../snippets/src/main/java - - - - - add-snippets-tests - - add-test-source - - - - ../snippets/src/test/java - - - - - - - - diff --git a/library_generation/owlbot/templates/java_library/samples/pom.xml b/library_generation/owlbot/templates/java_library/samples/pom.xml deleted file mode 100644 index 5fc52b1526..0000000000 --- a/library_generation/owlbot/templates/java_library/samples/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-{{metadata['repo']['name']}}-samples - 0.0.1-SNAPSHOT - pom - Google {{metadata['repo']['name_pretty']}} Samples Parent - https://github.com/{{metadata['repo']['repo']}} - - Java idiomatic client for Google Cloud Platform services. - - - - - com.google.cloud.samples - shared-configuration - 1.2.2 - - - - 1.8 - 1.8 - UTF-8 - - - - install-without-bom - snapshot - snippets - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.1.3 - - true - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.7.0 - - true - - - - - diff --git a/library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml b/library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml deleted file mode 100644 index 62a83b440e..0000000000 --- a/library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml +++ /dev/null @@ -1,85 +0,0 @@ -{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} -{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} - - - 4.0.0 - com.google.cloud - {{metadata['repo']['name']}}-snapshot - jar - Google {{metadata['repo']['name_pretty']}} Snapshot Samples - https://github.com/{{metadata['repo']['repo']}} - - - - com.google.cloud.samples - shared-configuration - 1.2.0 - - - - 1.8 - 1.8 - UTF-8 - - - - - - {{ group_id }} - {{ artifact_id }} - {{ metadata['latest_version'] }} - - - - - junit - junit - 4.13.2 - test - - - com.google.truth - truth - 1.1.3 - test - - - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.3.0 - - - add-snippets-source - - add-source - - - - ../snippets/src/main/java - - - - - add-snippets-tests - - add-test-source - - - - ../snippets/src/test/java - - - - - - - - diff --git a/library_generation/owlbot/templates/java_library/samples/snippets/pom.xml b/library_generation/owlbot/templates/java_library/samples/snippets/pom.xml deleted file mode 100644 index c6b9981507..0000000000 --- a/library_generation/owlbot/templates/java_library/samples/snippets/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ -{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} -{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} - - - 4.0.0 - com.google.cloud - {{metadata['repo']['name']}}-snippets - jar - Google {{metadata['repo']['name_pretty']}} Snippets - https://github.com/{{metadata['repo']['repo']}} - - - - com.google.cloud.samples - shared-configuration - 1.2.0 - - - - 1.8 - 1.8 - UTF-8 - - - - - - {{ group_id }} - {{ artifact_id }} - {{ metadata['latest_version'] }} - - - - junit - junit - 4.13.2 - test - - - com.google.truth - truth - 1.1.3 - test - - - diff --git a/library_generation/owlbot/templates/poms/bom_pom.xml.j2 b/library_generation/owlbot/templates/poms/bom_pom.xml.j2 deleted file mode 100644 index ddcef5226f..0000000000 --- a/library_generation/owlbot/templates/poms/bom_pom.xml.j2 +++ /dev/null @@ -1,41 +0,0 @@ - - - 4.0.0 - {{main_module.group_id}} - {{main_module.artifact_id}}-bom - {{main_module.version}} - pom - {% if monorepo -%} - - com.google.cloud - google-cloud-pom-parent - {{ monorepo_version }} - ../../google-cloud-pom-parent/pom.xml - - {%- else -%} - - com.google.cloud - google-cloud-shared-config - 1.5.3 - - {%- endif %} - - Google {{name}} BOM - - BOM for {{name}} - - - - true - - - - {% for module in modules %} - - {{module.group_id}} - {{module.artifact_id}} - {{module.version}} - {% endfor %} - - - diff --git a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 deleted file mode 100644 index d0ae8cd7c5..0000000000 --- a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 +++ /dev/null @@ -1,156 +0,0 @@ - - - 4.0.0 - {{module.group_id}} - {{module.artifact_id}} - {{module.version}} - jar - Google {{name}} - {%- if not monorepo %} - https://github.com/{{repo}} - {%- endif %} - {{name}} {{description}} - - {{parent_module.group_id}} - {{parent_module.artifact_id}} - {{parent_module.version}} - - - {{module.artifact_id}} - - - - io.grpc - grpc-api - - - io.grpc - grpc-stub - - - io.grpc - grpc-protobuf - - - com.google.api - api-common - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - -{% for module in proto_modules %} - - {{module.group_id}} - {{module.artifact_id}} - {% endfor %} - - com.google.guava - guava - - - com.google.api - gax - - - com.google.api - gax-grpc - - - com.google.api - gax-httpjson - - - com.google.api.grpc - proto-google-iam-v1 - -{%- if not monorepo %} - - com.google.api.grpc - grpc-google-common-protos - - - com.google.api.grpc - grpc-google-iam-v1 - -{%- endif %} - - org.threeten - threetenbp - - - -{%- if monorepo %} - - com.google.api.grpc - grpc-google-common-protos - test - - - com.google.api.grpc - grpc-google-iam-v1 - test - -{%- endif %} - - junit - junit - test - -{% for module in grpc_modules %} - - {{module.group_id}} - {{module.artifact_id}} - test - {% endfor %} - - - com.google.api - gax - testlib - test - - - com.google.api - gax-grpc - testlib - test - - - com.google.api - gax-httpjson - testlib - test - - - - {%- if not monorepo %} - - - java9 - - [9,) - - - - javax.annotation - javax.annotation-api - - - - - - - - - org.codehaus.mojo - flatten-maven-plugin - - - - {%- endif %} - diff --git a/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 b/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 deleted file mode 100644 index 514861e7a7..0000000000 --- a/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 +++ /dev/null @@ -1,71 +0,0 @@ - - 4.0.0 - {{module.group_id}} - {{module.artifact_id}} - {{module.version}} - {{module.artifact_id}} - GRPC library for {{main_module.artifact_id}} - - {{parent_module.group_id}} - {{parent_module.artifact_id}} - {{parent_module.version}} - - - - io.grpc - grpc-api - - - io.grpc - grpc-stub - - - io.grpc - grpc-protobuf - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - {{proto_module.group_id}} - {{proto_module.artifact_id}} - - - com.google.guava - guava - - - - {%- if not monorepo %} - - - java9 - - [9,) - - - - javax.annotation - javax.annotation-api - - - - - - - - - org.codehaus.mojo - flatten-maven-plugin - - - - {%- endif %} - diff --git a/library_generation/owlbot/templates/poms/parent_pom.xml.j2 b/library_generation/owlbot/templates/poms/parent_pom.xml.j2 deleted file mode 100644 index dcf922340e..0000000000 --- a/library_generation/owlbot/templates/poms/parent_pom.xml.j2 +++ /dev/null @@ -1,51 +0,0 @@ - - - 4.0.0 - {{main_module.group_id}} - {{main_module.artifact_id}}-parent - pom - {{main_module.version}} - Google {{name}} Parent - - Java idiomatic client for Google Cloud Platform services. - - - {% if monorepo -%} - - com.google.cloud - google-cloud-jar-parent - {{ monorepo_version }} - ../google-cloud-jar-parent/pom.xml - - {%- else -%} - - com.google.cloud - google-cloud-shared-config - 1.5.3 - - {%- endif %} - - - UTF-8 - UTF-8 - github - {{main_module.artifact_id}}-parent - - - - -{% for module in modules %} - {{module.group_id}} - {{module.artifact_id}} - {{module.version}} - -{% endfor %} - - - - -{% for module in modules %} {{module.artifact_id}} -{% endfor %} {{main_module.artifact_id}}-bom - - - diff --git a/library_generation/owlbot/templates/poms/proto_pom.xml.j2 b/library_generation/owlbot/templates/poms/proto_pom.xml.j2 deleted file mode 100644 index 886cd02663..0000000000 --- a/library_generation/owlbot/templates/poms/proto_pom.xml.j2 +++ /dev/null @@ -1,48 +0,0 @@ - - 4.0.0 - {{module.group_id}} - {{module.artifact_id}} - {{module.version}} - {{module.artifact_id}} - Proto library for {{main_module.artifact_id}} - - {{parent_module.group_id}} - {{parent_module.artifact_id}} - {{parent_module.version}} - - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - com.google.api.grpc - proto-google-iam-v1 - - - com.google.api - api-common - - - com.google.guava - guava - - - - {%- if not monorepo %} - - - - org.codehaus.mojo - flatten-maven-plugin - - - - {%- endif %} - diff --git a/library_generation/owlbot/templates/poms/versions.txt.j2 b/library_generation/owlbot/templates/poms/versions.txt.j2 deleted file mode 100644 index 2ebaf85d34..0000000000 --- a/library_generation/owlbot/templates/poms/versions.txt.j2 +++ /dev/null @@ -1,4 +0,0 @@ -# Format: -# module:released-version:current-version -{% for module in modules %} -{{module.artifact_id}}:{% if module.release_version %}{{module.release_version}}{% else %}{{module.version}}{% endif %}:{{module.version}}{% endfor %} diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh deleted file mode 100755 index eeec07156e..0000000000 --- a/library_generation/postprocess_library.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash -# -# Main functions to interact with owlbot post-processor - -# Runs the java owlbot post-processor. The resulting post-processed -# library gets stored in the $postprocessing_target argument -# Arguments -# 1 - postprocessing_target: path where the postprocessor will run. This folder -# has the following requirements -# - a .repo-metadata.json file must be present -# - an owlbot.py file must be present -# - an .OwlBot-hermetic.yaml file must be present -# 2 - preprocessed_sources_path: used to transfer the raw grpc, proto and gapic -# libraries into the postprocessing_target via copy-code -# 3 - versions_file: path to file containing versions to be applied to the poms -# 4 - owlbot_cli_source_folder: alternative folder with a structure exactly like -# googleapis-gen. It will be used instead of preprocessed_sources_path if -# 5 - is_monorepo: whether this library is a monorepo, which implies slightly -# different logic -# 6 - libraries_bom_version: used by our implementation of owlbot to render the -# readme -# 7 - library_version: used by our implementation of owlbot to render the -# readme -set -exo pipefail -scripts_root=$(dirname "$(readlink -f "$0")") - -postprocessing_target=$1 -preprocessed_sources_path=$2 -versions_file=$3 -owlbot_cli_source_folder=$4 -is_monorepo=$5 -libraries_bom_version=$6 -library_version=$7 -owlbot_yaml_file_name=".OwlBot-hermetic.yaml" - -source "${scripts_root}"/utils/utilities.sh - -declare -a required_inputs=("postprocessing_target" "versions_file" "is_monorepo") -for required_input in "${required_inputs[@]}"; do - if [[ -z "${!required_input}" ]]; then - echo "missing required ${required_input} argument, please specify one" - exit 1 - fi -done - -for owlbot_file in ".repo-metadata.json" "owlbot.py" "${owlbot_yaml_file_name}" -do - if [[ $(find "${postprocessing_target}" -name "${owlbot_file}" | wc -l) -eq 0 ]]; then - echo "necessary file for postprocessing '${owlbot_file}' was not found in postprocessing_target" - echo "please provide a postprocessing_target folder that is compatible with the OwlBot Java postprocessor" - exit 1 - fi -done - -if [[ -z "${owlbot_cli_source_folder}" ]]; then - owlbot_cli_source_folder=$(mktemp -d) - build_owlbot_cli_source_folder "${postprocessing_target}" "${owlbot_cli_source_folder}" "${preprocessed_sources_path}" -fi - - -# we determine the location of the .OwlBot-hermetic.yaml file by checking if the target -# folder is a monorepo folder or not -if [[ "${is_monorepo}" == "true" ]]; then - # the deep-remove-regex and deep-preserve-regex of the .OwlBot-hermetic.yaml - # files in the monorepo libraries assume that `copy-code` is run - # from the root of the monorepo. However, we call `copy-code` from inside each - # library, so a path like `/java-asset/google-.*/src` will not have - # any effect. We solve this by creating a temporary owlbot yaml with - # the patched paths. - # For example, we convert - # - "/java-asset/google-.*/src" - # to - # - "/google-.*/src" - - library_name=$(basename "${postprocessing_target}") - cat "${postprocessing_target}/${owlbot_yaml_file_name}" \ - | sed "s/- \"\/${library_name}/ - \"/" \ - > "${postprocessing_target}/.OwlBot.hermetic.yaml" - owlbot_yaml_relative_path=".OwlBot.hermetic.yaml" -else - owlbot_yaml_relative_path=".github/${owlbot_yaml_file_name}" -fi - -# Default values for running copy-code directly from host -repo_workspace="/workspace" -preprocessed_libraries_binding="${owlbot_cli_source_folder}" - -pushd "${postprocessing_target}" - -owl-bot copy-code \ - --source-repo-commit-hash=none \ - --source-repo="${owlbot_cli_source_folder}" \ - --config-file="${owlbot_yaml_relative_path}" - - -# clean the custom owlbot yaml -if [[ "${is_monorepo}" == "true" ]]; then - rm "${postprocessing_target}/.OwlBot.hermetic.yaml" -fi - -# run the postprocessor -echo 'running owl-bot post-processor' -pushd "${postprocessing_target}" -bash "${scripts_root}/owlbot/bin/entrypoint.sh" \ - "${scripts_root}" \ - "${versions_file}" \ - "${is_monorepo}" \ - "${libraries_bom_version}" \ - "${library_version}" - -popd # postprocessing_target diff --git a/library_generation/requirements.in b/library_generation/requirements.in deleted file mode 100644 index e5e6487246..0000000000 --- a/library_generation/requirements.in +++ /dev/null @@ -1,29 +0,0 @@ -# add new packages here and run `pip-compile requirements.in --generate-hashes` to -# generate an entry with hashes in requirements.txt -absl-py==2.1.0 -attr==0.3.2 -attrs==23.2.0 -black==24.8.0 -click==8.1.7 -gitdb==4.0.11 -GitPython==3.1.43 -lxml==5.3.0 -MarkupSafe==2.1.5 -mypy-extensions==1.0.0 -packaging==23.2 -pathspec==0.12.1 -PyYAML==6.0.2 -smmap==5.0.1 -typing==3.7.4.3 -parameterized==0.9.0 # used in parameterized test -colorlog==6.8.2 -watchdog==4.0.2 -nox==2024.4.15 -requests==2.32.3 -setuptools==65.5.1 -jinja2==3.1.4 -# typing-extensions is a transitive dependency. If we run `pip-compile ... --generate-hashes` it will produce -# a list where typing extensions is pinned to >=4.0.1. This will produce an error saying "all requirements -# must have their versions pinned with ==". The following line pins the dependency to a specific version via == -typing-extensions==4.0.1 -requests-mock # used in owlbot unit tests diff --git a/library_generation/requirements.txt b/library_generation/requirements.txt deleted file mode 100644 index 5407b68e05..0000000000 --- a/library_generation/requirements.txt +++ /dev/null @@ -1,546 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --generate-hashes library_generation/requirements.in -# -absl-py==2.1.0 \ - --hash=sha256:526a04eadab8b4ee719ce68f204172ead1027549089702d99b9059f129ff1308 \ - --hash=sha256:7820790efbb316739cde8b4e19357243fc3608a152024288513dd968d7d959ff - # via -r library_generation/requirements.in -argcomplete==3.5.0 \ - --hash=sha256:4349400469dccfb7950bb60334a680c58d88699bff6159df61251878dc6bf74b \ - --hash=sha256:d4bcf3ff544f51e16e54228a7ac7f486ed70ebf2ecfe49a63a91171c76bf029b - # via nox -attr==0.3.2 \ - --hash=sha256:1ceebca768181cdcce9827611b1d728e592be5d293911539ea3d0b0bfa1146f4 \ - --hash=sha256:4f4bffeea8c27387bde446675a7ac24f3b8fea1075f12d849b5f5c5181fc8336 - # via -r library_generation/requirements.in -attrs==23.2.0 \ - --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ - --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 - # via -r library_generation/requirements.in -black==24.8.0 \ - --hash=sha256:09cdeb74d494ec023ded657f7092ba518e8cf78fa8386155e4a03fdcc44679e6 \ - --hash=sha256:1f13f7f386f86f8121d76599114bb8c17b69d962137fc70efe56137727c7047e \ - --hash=sha256:2500945420b6784c38b9ee885af039f5e7471ef284ab03fa35ecdde4688cd83f \ - --hash=sha256:2b59b250fdba5f9a9cd9d0ece6e6d993d91ce877d121d161e4698af3eb9c1018 \ - --hash=sha256:3c4285573d4897a7610054af5a890bde7c65cb466040c5f0c8b732812d7f0e5e \ - --hash=sha256:505289f17ceda596658ae81b61ebbe2d9b25aa78067035184ed0a9d855d18afd \ - --hash=sha256:62e8730977f0b77998029da7971fa896ceefa2c4c4933fcd593fa599ecbf97a4 \ - --hash=sha256:649f6d84ccbae73ab767e206772cc2d7a393a001070a4c814a546afd0d423aed \ - --hash=sha256:6e55d30d44bed36593c3163b9bc63bf58b3b30e4611e4d88a0c3c239930ed5b2 \ - --hash=sha256:707a1ca89221bc8a1a64fb5e15ef39cd755633daa672a9db7498d1c19de66a42 \ - --hash=sha256:72901b4913cbac8972ad911dc4098d5753704d1f3c56e44ae8dce99eecb0e3af \ - --hash=sha256:73bbf84ed136e45d451a260c6b73ed674652f90a2b3211d6a35e78054563a9bb \ - --hash=sha256:7c046c1d1eeb7aea9335da62472481d3bbf3fd986e093cffd35f4385c94ae368 \ - --hash=sha256:81c6742da39f33b08e791da38410f32e27d632260e599df7245cccee2064afeb \ - --hash=sha256:837fd281f1908d0076844bc2b801ad2d369c78c45cf800cad7b61686051041af \ - --hash=sha256:972085c618ee94f402da1af548a4f218c754ea7e5dc70acb168bfaca4c2542ed \ - --hash=sha256:9e84e33b37be070ba135176c123ae52a51f82306def9f7d063ee302ecab2cf47 \ - --hash=sha256:b19c9ad992c7883ad84c9b22aaa73562a16b819c1d8db7a1a1a49fb7ec13c7d2 \ - --hash=sha256:d6417535d99c37cee4091a2f24eb2b6d5ec42b144d50f1f2e436d9fe1916fe1a \ - --hash=sha256:eab4dd44ce80dea27dc69db40dab62d4ca96112f87996bca68cd75639aeb2e4c \ - --hash=sha256:f490dbd59680d809ca31efdae20e634f3fae27fba3ce0ba3208333b713bc3920 \ - --hash=sha256:fb6e2c0b86bbd43dee042e48059c9ad7830abd5c94b0bc518c0eeec57c3eddc1 - # via -r library_generation/requirements.in -certifi==2024.8.30 \ - --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \ - --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9 - # via requests -charset-normalizer==3.3.2 \ - --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ - --hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \ - --hash=sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786 \ - --hash=sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8 \ - --hash=sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09 \ - --hash=sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185 \ - --hash=sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574 \ - --hash=sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e \ - --hash=sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519 \ - --hash=sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898 \ - --hash=sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269 \ - --hash=sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3 \ - --hash=sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f \ - --hash=sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6 \ - --hash=sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8 \ - --hash=sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a \ - --hash=sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73 \ - --hash=sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc \ - --hash=sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714 \ - --hash=sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2 \ - --hash=sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc \ - --hash=sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce \ - --hash=sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d \ - --hash=sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e \ - --hash=sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6 \ - --hash=sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269 \ - --hash=sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96 \ - --hash=sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d \ - --hash=sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a \ - --hash=sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4 \ - --hash=sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77 \ - --hash=sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d \ - --hash=sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0 \ - --hash=sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed \ - --hash=sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068 \ - --hash=sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac \ - --hash=sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25 \ - --hash=sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8 \ - --hash=sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab \ - --hash=sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26 \ - --hash=sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2 \ - --hash=sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db \ - --hash=sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f \ - --hash=sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5 \ - --hash=sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99 \ - --hash=sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c \ - --hash=sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d \ - --hash=sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811 \ - --hash=sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa \ - --hash=sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a \ - --hash=sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03 \ - --hash=sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b \ - --hash=sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04 \ - --hash=sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c \ - --hash=sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001 \ - --hash=sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458 \ - --hash=sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389 \ - --hash=sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99 \ - --hash=sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985 \ - --hash=sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537 \ - --hash=sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238 \ - --hash=sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f \ - --hash=sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d \ - --hash=sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796 \ - --hash=sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a \ - --hash=sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143 \ - --hash=sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8 \ - --hash=sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c \ - --hash=sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5 \ - --hash=sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5 \ - --hash=sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711 \ - --hash=sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4 \ - --hash=sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6 \ - --hash=sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c \ - --hash=sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7 \ - --hash=sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4 \ - --hash=sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b \ - --hash=sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae \ - --hash=sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12 \ - --hash=sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c \ - --hash=sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae \ - --hash=sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8 \ - --hash=sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887 \ - --hash=sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b \ - --hash=sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4 \ - --hash=sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f \ - --hash=sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5 \ - --hash=sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33 \ - --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ - --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 - # via requests -click==8.1.7 \ - --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ - --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de - # via - # -r library_generation/requirements.in - # black -colorlog==6.8.2 \ - --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ - --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33 - # via - # -r library_generation/requirements.in - # nox -distlib==0.3.8 \ - --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ - --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 - # via virtualenv -filelock==3.16.1 \ - --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \ - --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435 - # via virtualenv -gitdb==4.0.11 \ - --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ - --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b - # via - # -r library_generation/requirements.in - # gitpython -gitpython==3.1.43 \ - --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ - --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff - # via -r library_generation/requirements.in -idna==3.10 \ - --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ - --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 - # via requests -jinja2==3.1.4 \ - --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ - --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d - # via -r library_generation/requirements.in -lxml==5.3.0 \ - --hash=sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e \ - --hash=sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229 \ - --hash=sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3 \ - --hash=sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5 \ - --hash=sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70 \ - --hash=sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15 \ - --hash=sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002 \ - --hash=sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd \ - --hash=sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22 \ - --hash=sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf \ - --hash=sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22 \ - --hash=sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832 \ - --hash=sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727 \ - --hash=sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e \ - --hash=sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30 \ - --hash=sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f \ - --hash=sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f \ - --hash=sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51 \ - --hash=sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4 \ - --hash=sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de \ - --hash=sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875 \ - --hash=sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42 \ - --hash=sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e \ - --hash=sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6 \ - --hash=sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391 \ - --hash=sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc \ - --hash=sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b \ - --hash=sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237 \ - --hash=sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4 \ - --hash=sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86 \ - --hash=sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f \ - --hash=sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a \ - --hash=sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8 \ - --hash=sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f \ - --hash=sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903 \ - --hash=sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03 \ - --hash=sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e \ - --hash=sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99 \ - --hash=sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7 \ - --hash=sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab \ - --hash=sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d \ - --hash=sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22 \ - --hash=sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492 \ - --hash=sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b \ - --hash=sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3 \ - --hash=sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be \ - --hash=sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469 \ - --hash=sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f \ - --hash=sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a \ - --hash=sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c \ - --hash=sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a \ - --hash=sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4 \ - --hash=sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94 \ - --hash=sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442 \ - --hash=sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b \ - --hash=sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84 \ - --hash=sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c \ - --hash=sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9 \ - --hash=sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1 \ - --hash=sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be \ - --hash=sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367 \ - --hash=sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e \ - --hash=sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21 \ - --hash=sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa \ - --hash=sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16 \ - --hash=sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d \ - --hash=sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe \ - --hash=sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83 \ - --hash=sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba \ - --hash=sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040 \ - --hash=sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763 \ - --hash=sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8 \ - --hash=sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff \ - --hash=sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2 \ - --hash=sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a \ - --hash=sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b \ - --hash=sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce \ - --hash=sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c \ - --hash=sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577 \ - --hash=sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8 \ - --hash=sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71 \ - --hash=sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512 \ - --hash=sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540 \ - --hash=sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f \ - --hash=sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2 \ - --hash=sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a \ - --hash=sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce \ - --hash=sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e \ - --hash=sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2 \ - --hash=sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27 \ - --hash=sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1 \ - --hash=sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d \ - --hash=sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1 \ - --hash=sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330 \ - --hash=sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920 \ - --hash=sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99 \ - --hash=sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff \ - --hash=sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18 \ - --hash=sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff \ - --hash=sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c \ - --hash=sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179 \ - --hash=sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080 \ - --hash=sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19 \ - --hash=sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d \ - --hash=sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70 \ - --hash=sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32 \ - --hash=sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a \ - --hash=sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2 \ - --hash=sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79 \ - --hash=sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3 \ - --hash=sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5 \ - --hash=sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f \ - --hash=sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d \ - --hash=sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3 \ - --hash=sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b \ - --hash=sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753 \ - --hash=sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9 \ - --hash=sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957 \ - --hash=sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033 \ - --hash=sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb \ - --hash=sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656 \ - --hash=sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab \ - --hash=sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b \ - --hash=sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d \ - --hash=sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd \ - --hash=sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859 \ - --hash=sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11 \ - --hash=sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c \ - --hash=sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a \ - --hash=sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005 \ - --hash=sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654 \ - --hash=sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80 \ - --hash=sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e \ - --hash=sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec \ - --hash=sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7 \ - --hash=sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965 \ - --hash=sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945 \ - --hash=sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8 - # via -r library_generation/requirements.in -markupsafe==2.1.5 \ - --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ - --hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \ - --hash=sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f \ - --hash=sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3 \ - --hash=sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532 \ - --hash=sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f \ - --hash=sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617 \ - --hash=sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df \ - --hash=sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4 \ - --hash=sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906 \ - --hash=sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f \ - --hash=sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4 \ - --hash=sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8 \ - --hash=sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371 \ - --hash=sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2 \ - --hash=sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465 \ - --hash=sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52 \ - --hash=sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6 \ - --hash=sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169 \ - --hash=sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad \ - --hash=sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2 \ - --hash=sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0 \ - --hash=sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029 \ - --hash=sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f \ - --hash=sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a \ - --hash=sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced \ - --hash=sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5 \ - --hash=sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c \ - --hash=sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf \ - --hash=sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9 \ - --hash=sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb \ - --hash=sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad \ - --hash=sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3 \ - --hash=sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1 \ - --hash=sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46 \ - --hash=sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc \ - --hash=sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a \ - --hash=sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee \ - --hash=sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900 \ - --hash=sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5 \ - --hash=sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea \ - --hash=sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f \ - --hash=sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5 \ - --hash=sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e \ - --hash=sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a \ - --hash=sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f \ - --hash=sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50 \ - --hash=sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a \ - --hash=sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b \ - --hash=sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4 \ - --hash=sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff \ - --hash=sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2 \ - --hash=sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46 \ - --hash=sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b \ - --hash=sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf \ - --hash=sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5 \ - --hash=sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5 \ - --hash=sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab \ - --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \ - --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68 - # via - # -r library_generation/requirements.in - # jinja2 -mypy-extensions==1.0.0 \ - --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ - --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 - # via - # -r library_generation/requirements.in - # black -nox==2024.4.15 \ - --hash=sha256:6492236efa15a460ecb98e7b67562a28b70da006ab0be164e8821177577c0565 \ - --hash=sha256:ecf6700199cdfa9e5ea0a41ff5e6ef4641d09508eda6edb89d9987864115817f - # via -r library_generation/requirements.in -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 - # via - # -r library_generation/requirements.in - # black - # nox -parameterized==0.9.0 \ - --hash=sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b \ - --hash=sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1 - # via -r library_generation/requirements.in -pathspec==0.12.1 \ - --hash=sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 \ - --hash=sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712 - # via - # -r library_generation/requirements.in - # black -platformdirs==4.3.6 \ - --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ - --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb - # via - # black - # virtualenv -pyyaml==6.0.2 \ - --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \ - --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ - --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \ - --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \ - --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ - --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \ - --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ - --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \ - --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ - --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \ - --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \ - --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \ - --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \ - --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \ - --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \ - --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \ - --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ - --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \ - --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ - --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \ - --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \ - --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \ - --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \ - --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ - --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ - --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \ - --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \ - --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ - --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ - --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \ - --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ - --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \ - --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ - --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \ - --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \ - --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ - --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \ - --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \ - --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \ - --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ - --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ - --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \ - --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ - --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ - --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \ - --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \ - --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \ - --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \ - --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \ - --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \ - --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \ - --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ - --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 - # via -r library_generation/requirements.in -requests==2.32.3 \ - --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ - --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 - # via - # -r library_generation/requirements.in - # requests-mock -requests-mock==1.12.1 \ - --hash=sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563 \ - --hash=sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401 - # via -r library_generation/requirements.in -smmap==5.0.1 \ - --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ - --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da - # via - # -r library_generation/requirements.in - # gitdb -typing==3.7.4.3 \ - --hash=sha256:1187fb9c82fd670d10aa07bbb6cfcfe4bdda42d6fab8d5134f04e8c4d0b71cc9 \ - --hash=sha256:283d868f5071ab9ad873e5e52268d611e851c870a2ba354193026f2dfb29d8b5 - # via -r library_generation/requirements.in -typing-extensions==4.0.1 \ - --hash=sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e \ - --hash=sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b - # via -r library_generation/requirements.in -urllib3==2.2.3 \ - --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \ - --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9 - # via requests -virtualenv==20.26.6 \ - --hash=sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48 \ - --hash=sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2 - # via nox -watchdog==4.0.2 \ - --hash=sha256:0b4359067d30d5b864e09c8597b112fe0a0a59321a0f331498b013fb097406b4 \ - --hash=sha256:0d8a7e523ef03757a5aa29f591437d64d0d894635f8a50f370fe37f913ce4e19 \ - --hash=sha256:0e83619a2d5d436a7e58a1aea957a3c1ccbf9782c43c0b4fed80580e5e4acd1a \ - --hash=sha256:10b6683df70d340ac3279eff0b2766813f00f35a1d37515d2c99959ada8f05fa \ - --hash=sha256:132937547a716027bd5714383dfc40dc66c26769f1ce8a72a859d6a48f371f3a \ - --hash=sha256:1cdcfd8142f604630deef34722d695fb455d04ab7cfe9963055df1fc69e6727a \ - --hash=sha256:2d468028a77b42cc685ed694a7a550a8d1771bb05193ba7b24006b8241a571a1 \ - --hash=sha256:32be97f3b75693a93c683787a87a0dc8db98bb84701539954eef991fb35f5fbc \ - --hash=sha256:770eef5372f146997638d737c9a3c597a3b41037cfbc5c41538fc27c09c3a3f9 \ - --hash=sha256:7c7d4bf585ad501c5f6c980e7be9c4f15604c7cc150e942d82083b31a7548930 \ - --hash=sha256:88456d65f207b39f1981bf772e473799fcdc10801062c36fd5ad9f9d1d463a73 \ - --hash=sha256:914285126ad0b6eb2258bbbcb7b288d9dfd655ae88fa28945be05a7b475a800b \ - --hash=sha256:936acba76d636f70db8f3c66e76aa6cb5136a936fc2a5088b9ce1c7a3508fc83 \ - --hash=sha256:980b71510f59c884d684b3663d46e7a14b457c9611c481e5cef08f4dd022eed7 \ - --hash=sha256:984306dc4720da5498b16fc037b36ac443816125a3705dfde4fd90652d8028ef \ - --hash=sha256:a2cffa171445b0efa0726c561eca9a27d00a1f2b83846dbd5a4f639c4f8ca8e1 \ - --hash=sha256:aa160781cafff2719b663c8a506156e9289d111d80f3387cf3af49cedee1f040 \ - --hash=sha256:b2c45f6e1e57ebb4687690c05bc3a2c1fb6ab260550c4290b8abb1335e0fd08b \ - --hash=sha256:b4dfbb6c49221be4535623ea4474a4d6ee0a9cef4a80b20c28db4d858b64e270 \ - --hash=sha256:baececaa8edff42cd16558a639a9b0ddf425f93d892e8392a56bf904f5eff22c \ - --hash=sha256:bcfd02377be80ef3b6bc4ce481ef3959640458d6feaae0bd43dd90a43da90a7d \ - --hash=sha256:c0b14488bd336c5b1845cee83d3e631a1f8b4e9c5091ec539406e4a324f882d8 \ - --hash=sha256:c100d09ac72a8a08ddbf0629ddfa0b8ee41740f9051429baa8e31bb903ad7508 \ - --hash=sha256:c344453ef3bf875a535b0488e3ad28e341adbd5a9ffb0f7d62cefacc8824ef2b \ - --hash=sha256:c50f148b31b03fbadd6d0b5980e38b558046b127dc483e5e4505fcef250f9503 \ - --hash=sha256:c82253cfc9be68e3e49282831afad2c1f6593af80c0daf1287f6a92657986757 \ - --hash=sha256:cd67c7df93eb58f360c43802acc945fa8da70c675b6fa37a241e17ca698ca49b \ - --hash=sha256:d7ab624ff2f663f98cd03c8b7eedc09375a911794dfea6bf2a359fcc266bff29 \ - --hash=sha256:e252f8ca942a870f38cf785aef420285431311652d871409a64e2a0a52a2174c \ - --hash=sha256:ede7f010f2239b97cc79e6cb3c249e72962404ae3865860855d5cbe708b0fd22 \ - --hash=sha256:eeea812f38536a0aa859972d50c76e37f4456474b02bd93674d1947cf1e39578 \ - --hash=sha256:f15edcae3830ff20e55d1f4e743e92970c847bcddc8b7509bcd172aa04de506e \ - --hash=sha256:f5315a8c8dd6dd9425b974515081fc0aadca1d1d61e078d2246509fd756141ee \ - --hash=sha256:f6ee8dedd255087bc7fe82adf046f0b75479b989185fb0bdf9a98b612170eac7 \ - --hash=sha256:f7c739888c20f99824f7aa9d31ac8a97353e22d0c0e54703a547a218f6637eb3 - # via -r library_generation/requirements.in - -# WARNING: The following packages were not pinned, but pip requires them to be -# pinned when the requirements file includes hashes and the requirement is not -# satisfied by a package already installed. Consider using the --allow-unsafe flag. -# setuptools diff --git a/library_generation/setup.py b/library_generation/setup.py deleted file mode 100755 index 9f78194241..0000000000 --- a/library_generation/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Package information of library_generation python scripts -""" - -from setuptools import setup - -setup( - name="library_generation", - version="0.1", - package_dir={ - "library_generation": ".", - "synthtool": "owlbot/synthtool", - }, - package_data={ - "library_generation": [ - "generate_library.sh", - "postprocess_library.sh", - "utils/utilities.sh", - "templates/*.j2", - "gapic-generator-java-wrapper", - "requirements.*", - "owlbot/bin/*.sh", - "owlbot/src/*.py", - "owlbot/src/poms/*.py", - "owlbot/templates/clirr/*.j2", - "owlbot/templates/poms/*.j2", - "owlbot/templates/java_library/**/*", - ], - "synthtool": ["owlbot/synthtool/**/*"], - }, -) diff --git a/library_generation/templates/gapic-libraries-bom.xml.j2 b/library_generation/templates/gapic-libraries-bom.xml.j2 deleted file mode 100644 index 45dbdf42ce..0000000000 --- a/library_generation/templates/gapic-libraries-bom.xml.j2 +++ /dev/null @@ -1,37 +0,0 @@ - - - 4.0.0 - com.google.cloud - gapic-libraries-bom - pom - {{ monorepo_version }} - Google Cloud Java BOM - - BOM for the libraries in google-cloud-java repository. Users should not - depend on this artifact explicitly because this BOM is an implementation - detail of the Libraries BOM. - - - - google-cloud-pom-parent - com.google.cloud - {{ monorepo_version }} - ../google-cloud-pom-parent/pom.xml - - - - - {%- for bom_config in bom_configs %} - - {{ bom_config.group_id }} - {{ bom_config.artifact_id }} - {{ bom_config.version }} - {%- if bom_config.is_import %} - pom - import - {%- endif %} - - {%- endfor %} - - - diff --git a/library_generation/templates/owlbot.py.j2 b/library_generation/templates/owlbot.py.j2 deleted file mode 100644 index b3a7dee5a4..0000000000 --- a/library_generation/templates/owlbot.py.j2 +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import synthtool as s -{% if should_include_templates %}from synthtool.languages import java{% endif %} - - -for library in s.get_staging_dirs(): - # put any special-case replacements here - s.move(library) - -s.remove_staging_dirs() -{% if should_include_templates %}java.common_templates(monorepo=True, {% if template_excludes %}excludes=[ -{%- for exclude in template_excludes %} - "{{ exclude }}"{% if not loop.last %},{% endif %} -{%- endfor %} -]{% endif %}){% endif %} diff --git a/library_generation/templates/owlbot.yaml.monorepo.j2 b/library_generation/templates/owlbot.yaml.monorepo.j2 deleted file mode 100644 index 9ed63c4260..0000000000 --- a/library_generation/templates/owlbot.yaml.monorepo.j2 +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{% if artifact_id %} -deep-remove-regex: -- "/{{ module_name }}/grpc-google-.*/src" -- "/{{ module_name }}/proto-google-.*/src" -- "/{{ module_name }}/google-.*/src" -- "/{{ module_name }}/samples/snippets/generated" - -deep-preserve-regex: -- "/{{ module_name }}/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" - -deep-copy-regex: -- source: "/{{ proto_path }}/(v.*)/.*-java/proto-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_id }}-$1/src" -- source: "/{{ proto_path }}/(v.*)/.*-java/grpc-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_id }}-$1/src" -- source: "/{{ proto_path }}/(v.*)/.*-java/gapic-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_id }}/src" -- source: "/{{ proto_path }}/(v.*)/.*-java/samples/snippets/generated" - dest: "/owl-bot-staging/{{ module_name }}/$1/samples/snippets/generated" -{%- endif %} - -api-name: {{ api_shortname }} diff --git a/library_generation/templates/root-pom.xml.j2 b/library_generation/templates/root-pom.xml.j2 deleted file mode 100644 index 56704aa4e4..0000000000 --- a/library_generation/templates/root-pom.xml.j2 +++ /dev/null @@ -1,88 +0,0 @@ - - - 4.0.0 - google-cloud-java - com.google.cloud - 0.201.0 - pom - - - true - - - - gapic-libraries-bom - google-cloud-jar-parent - google-cloud-pom-parent - {%- for module in modules %} - {{ module }} - {%- endfor %} - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.1.3 - - true - - - - - - - - release-staging-repository - - - - !gpg.executable - - - - - sonatype-nexus-snapshots - https://google.oss.sonatype.org/content/repositories/snapshots - - - sonatype-nexus-staging - https://google.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.7.0 - true - - sonatype-nexus-staging - https://google.oss.sonatype.org/ - false - - - - - - - release-non-google-oss-sonatype - - - - org.sonatype.plugins - nexus-staging-maven-plugin - - ossrh - https://oss.sonatype.org/ - - - - - - - diff --git a/library_generation/test/__init__.py b/library_generation/test/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/cli/__init__.py b/library_generation/test/cli/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py deleted file mode 100644 index 80197ab2df..0000000000 --- a/library_generation/test/cli/entry_point_unit_tests.py +++ /dev/null @@ -1,416 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest -from unittest.mock import patch, ANY -from click.testing import CliRunner -from library_generation.cli.entry_point import ( - generate, - validate_generation_config, - __generate_repo_and_pr_description_impl as generate_impl, -) -from library_generation.model.generation_config import from_yaml - -script_dir = os.path.dirname(os.path.realpath(__file__)) -test_resource_dir = os.path.join(script_dir, "..", "resources", "test-config") - - -class EntryPointTest(unittest.TestCase): - def test_entry_point_without_config_raise_file_exception(self): - os.chdir(script_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke(generate, ["--repository-path=."]) - self.assertEqual(1, result.exit_code) - self.assertEqual(FileNotFoundError, result.exc_info[0]) - self.assertRegex( - result.exception.args[0], "generation_config.yaml does not exist." - ) - - def test_entry_point_with_baseline_without_current_raise_file_exception(self): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - generate, - [ - "--baseline-generation-config-path=path/to/config.yaml", - "--repository-path=.", - ], - ) - self.assertEqual(1, result.exit_code) - self.assertEqual(FileNotFoundError, result.exc_info[0]) - self.assertRegex( - result.exception.args[0], - "current_generation_config is not specified when " - "baseline_generation_config is specified.", - ) - - def test_validate_generation_config_succeeds( - self, - ): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - validate_generation_config, - [f"--generation-config-path={test_resource_dir}/generation_config.yaml"], - ) - self.assertEqual(0, result.exit_code) - - def test_validate_generation_config_with_duplicate_library_name_raise_file_exception( - self, - ): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - validate_generation_config, - [ - f"--generation-config-path={test_resource_dir}/generation_config_with_duplicate_library_name.yaml" - ], - ) - self.assertEqual(1, result.exit_code) - self.assertEqual(SystemExit, result.exc_info[0]) - self.assertRegex( - result.output, - "have the same library name", - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_without_changes_triggers_full_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of non monorepos - (HW libraries). generate() should call generate_from_yaml() - with target_library_names=None in order to trigger the full generation - """ - config_path = f"{test_resource_dir}/generation_config.yaml" - self.assertFalse(from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=None, - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_without_changes_with_includes_triggers_selective_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of non monorepos - (HW libraries). - generate() should call generate_from_yaml() with - target_library_names equals includes. - """ - config_path = f"{test_resource_dir}/generation_config.yaml" - self.assertFalse(from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names="cloudasset,non-existent-library", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["cloudasset", "non-existent-library"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_with_changes_triggers_full_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of non monorepos - (HW libraries). generate() should call generate_from_yaml() - with target_library_names=None in order to trigger the full generation - """ - baseline_config_path = f"{test_resource_dir}/generation_config.yaml" - current_config_path = ( - f"{test_resource_dir}/generation_config_library_modified.yaml" - ) - self.assertFalse(from_yaml(current_config_path).is_monorepo()) - self.assertFalse(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=None, - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_with_changes_with_includes_triggers_selective_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of non monorepos - (HW libraries). - generate() should call generate_from_yaml() with - target_library_names equals includes - """ - baseline_config_path = f"{test_resource_dir}/generation_config.yaml" - current_config_path = ( - f"{test_resource_dir}/generation_config_library_modified.yaml" - ) - self.assertFalse(from_yaml(current_config_path).is_monorepo()) - self.assertFalse(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names="cloudasset,non-existent-library", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["cloudasset", "non-existent-library"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_common_protos_triggers_full_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo with - common protos. - generate() should call generate_from_yaml() with - target_library_names=None in order to trigger the full generation - """ - config_path = f"{test_resource_dir}/monorepo_with_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=None, - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo with - common protos. - target_library_names is the same as includes. - """ - config_path = f"{test_resource_dir}/monorepo_with_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names="iam,non-existent-library", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["iam", "non-existent-library"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_without_change_does_not_trigger_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - generate() should call generate_from_yaml() with - target_library_names=changed libraries which does not trigger the full - generation. - """ - config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=[], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_without_change_with_includes_trigger_selective_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - generate() should call generate_from_yaml() with - target_library_names=changed libraries which does not trigger the full - generation. - """ - config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names="asset", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["asset"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_changed_config_without_includes_trigger_changed_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - target_library_names should be the changed libraries if includes - is not specified. - """ - current_config_path = f"{test_resource_dir}/monorepo_current.yaml" - baseline_config_path = f"{test_resource_dir}/monorepo_baseline.yaml" - self.assertTrue(from_yaml(current_config_path).is_monorepo()) - self.assertTrue(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["asset"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_changed_config_and_includes_trigger_selective_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - target_library_names should be the same as include libraries, regardless - the library exists or not. - """ - current_config_path = f"{test_resource_dir}/monorepo_current.yaml" - baseline_config_path = f"{test_resource_dir}/monorepo_baseline.yaml" - self.assertTrue(from_yaml(current_config_path).is_monorepo()) - self.assertTrue(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names="cloudbuild,non-existent-library", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["cloudbuild", "non-existent-library"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_without_changed_config_without_includes_does_not_trigger_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - target_library_names should be the changed libraries if includes - is not specified. - """ - config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=[], - ) diff --git a/library_generation/test/cli/generate_release_note_unit_tests.py b/library_generation/test/cli/generate_release_note_unit_tests.py deleted file mode 100644 index be0d6a4fed..0000000000 --- a/library_generation/test/cli/generate_release_note_unit_tests.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest -from click.testing import CliRunner -from library_generation.cli.generate_release_note import generate - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resource_dir = os.path.join(script_dir, "..", "resources", "test_generate_release_note") - - -class GenerateReleaseNoteTest(unittest.TestCase): - def test_gen_release_note_with_no_baseline_config_does_not_generate_note(self): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke(generate) - self.assertEqual(0, result.exit_code) - self.assertFalse(os.path.isfile("pr_description.txt")) - - def test_gen_release_note_with_no_current_config_does_not_generate_note(self): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - generate, ["--baseline-generation-config-path=any_config"] - ) - self.assertEqual(0, result.exit_code) - self.assertFalse(os.path.isfile("pr_description.txt")) - - def test_gen_release_note_with_nonexistent_baseline_config_does_not_generate_note( - self, - ): - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - generate, - [ - "--baseline-generation-config-path=non_existent_config", - "--current-generation-config-path=not_relevant", - ], - ) - self.assertEqual(0, result.exit_code) - self.assertFalse(os.path.isfile("pr_description.txt")) - - def test_gen_release_note_with_nonexistent_current_config_does_not_generate_note( - self, - ): - cwd = os.getcwd() - os.chdir(resource_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke( - generate, - [ - "--baseline-generation-config-path=empty_gen_config.yaml", - "--current-generation-config-path=non_existent_config", - ], - ) - self.assertEqual(0, result.exit_code) - self.assertFalse(os.path.isfile("pr_description.txt")) - os.chdir(cwd) diff --git a/library_generation/test/compare_poms.py b/library_generation/test/compare_poms.py deleted file mode 100644 index 95041651fb..0000000000 --- a/library_generation/test/compare_poms.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -Utility to compare the contents of two XML files. -This focuses on the tree structure of both XML files, meaning that element order and whitespace will be disregarded. -The only comparison points are: element path (e.g. project/dependencies) and element text -There is a special case for `dependency`, where the maven coordinates are prepared as well -""" - -from library_generation.utils.utilities import eprint -import xml.etree.ElementTree as et -from collections import Counter -import sys -import os - -current = os.path.dirname(os.path.realpath(__file__)) -parent = os.path.dirname(current) -sys.path.append(parent) - - -def get_text_from_element(node, element_name, namespace): - """ - Convenience method to access a node's child elements via path and get - its text. - """ - child = node.find(namespace + element_name) - return child.text if child is not None else "" - - -def print_counter(counter): - """ - Convenience method to pretty print the contents of a Counter (or dict) - """ - for key, value in counter.items(): - eprint(f"{key}: {value}") - - -def append_to_element_list(node, path, elements): - """ - Recursively traverses a node tree and appends element text to a given - `elements` array. If the element tag is `dependency` - then the maven coordinates for its children will be computed as well - """ - namespace_start, namespace_end, tag_name = node.tag.rpartition("}") - namespace = namespace_start + namespace_end - if tag_name == "dependency": - group_id = get_text_from_element(node, "groupId", namespace) - artifact_id = get_text_from_element(node, "artifactId", namespace) - artifact_str = "" - artifact_str += group_id - artifact_str += ":" + artifact_id - elements.append(path + "/" + tag_name + "=" + artifact_str) - if node.text and len(node.text.strip()) > 0: - elements.append(path + "/" + tag_name + "=" + node.text) - - if tag_name == "version": - # versions may be yet to be processed, we disregard them - return elements - - for child in node: - child_path = path + "/" + tag_name - append_to_element_list(child, child_path, elements) - - return elements - - -def compare_xml(expected, actual, print_trees): - """ - compares two XMLs for content differences - the argument print_whole_trees determines if both trees should be printed - """ - try: - expected_tree = et.parse(expected) - actual_tree = et.parse(actual) - except et.ParseError as e: - eprint(f"Error parsing XML") - raise e - except FileNotFoundError as e: - eprint(f"Error reading file") - raise e - - expected_elements = [] - actual_elements = [] - - append_to_element_list(expected_tree.getroot(), "/", expected_elements) - append_to_element_list(actual_tree.getroot(), "/", actual_elements) - - expected_counter = Counter(expected_elements) - actual_counter = Counter(actual_elements) - intersection = expected_counter & actual_counter - only_in_expected = expected_counter - intersection - only_in_actual = actual_counter - intersection - if print_trees: - eprint("expected") - print_counter(actual_counter) - eprint("actual") - print_counter(expected_counter) - if len(only_in_expected) > 0 or len(only_in_actual) > 0: - eprint("only in " + expected) - print_counter(only_in_expected) - eprint("only in " + actual) - print_counter(only_in_actual) - return True - return False - - -def compare_pom_in_subdir(base_dir: str, subdir: str) -> bool: - golden = os.path.join(base_dir, subdir, "pom-golden.xml") - pom = os.path.join(base_dir, subdir, "pom.xml") - return compare_xml( - golden, - pom, - False, - ) - - -if __name__ == "__main__": - if len(sys.argv) != 4: - eprint( - "Usage: python compare_xml.py " - ) - sys.exit(1) - - file1 = sys.argv[1] - file2 = sys.argv[2] - print_whole_trees = sys.argv[3] - has_diff = compare_xml(file1, file2, print_whole_trees) - - if has_diff: - eprint(f"The poms are different") - sys.exit(1) - eprint("The XML files are the same.") - sys.exit(0) diff --git a/library_generation/test/generate_library_unit_tests.py b/library_generation/test/generate_library_unit_tests.py deleted file mode 100644 index 7bc14e3e20..0000000000 --- a/library_generation/test/generate_library_unit_tests.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import subprocess -import unittest -import os -from library_generation.utils.utilities import ( - run_process_and_print_output as bash_call, - run_process_and_get_output_string as get_bash_call_output, -) - -script_dir = os.path.dirname(os.path.realpath(__file__)) - - -class GenerateLibraryUnitTests(unittest.TestCase): - """ - Confirms the correct behavior of `library_generation/utils/utilities.sh`. - - Note that there is an already existing, shell-based, test suite for - generate_library.sh, but these tests will soon be transferred to this one as - an effort to unify the implementation of the Hermetic Build scripts as - python-only. New tests for `utilities.sh` should be added in this file. - """ - - TEST_ARCHITECTURE = "linux-x86_64" - - def setUp(self): - # we create a simulated home folder that has a fake generator jar - # in its well-known location - self.simulated_home = get_bash_call_output("mktemp -d") - bash_call(f"mkdir {self.simulated_home}/.library_generation") - bash_call( - f"touch {self.simulated_home}/.library_generation/gapic-generator-java.jar" - ) - - # We create a per-test directory where all output files will be created into. - # Each folder will be deleted after its corresponding test finishes. - test_dir = get_bash_call_output("mktemp -d") - self.output_folder = self._run_command_and_get_sdout( - "get_output_folder", - cwd=test_dir, - ) - bash_call(f"mkdir {self.output_folder}") - - def tearDown(self): - bash_call(f"rm -rdf {self.simulated_home}") - - def _run_command(self, command, **kwargs): - env = os.environ.copy() - env["HOME"] = self.simulated_home - if "cwd" not in kwargs: - kwargs["cwd"] = self.output_folder - return bash_call( - [ - "bash", - "-exc", - f"source {script_dir}/../utils/utilities.sh " + f"&& {command}", - ], - exit_on_fail=False, - env=env, - **kwargs, - ) - - def _run_command_and_get_sdout(self, command, **kwargs): - return self._run_command( - command, stderr=subprocess.PIPE, **kwargs - ).stdout.decode()[:-1] - - def test_get_grpc_version_with_no_env_var_fails(self): - # the absence of DOCKER_GRPC_VERSION will make this function to fail - result = self._run_command("get_grpc_version") - self.assertEqual(1, result.returncode) - self.assertRegex(result.stdout.decode(), "DOCKER_GRPC_VERSION is not set") - - def test_get_protoc_version_with_no_env_var_fails(self): - # the absence of DOCKER_PROTOC_VERSION will make this function to fail - result = self._run_command("get_protoc_version") - self.assertEqual(1, result.returncode) - self.assertRegex(result.stdout.decode(), "DOCKER_PROTOC_VERSION is not set") - - def test_download_tools_without_baked_generator_fails(self): - # This test has the same structure as - # download_tools_succeed_with_baked_protoc, but meant for - # gapic-generator-java. - - test_protoc_version = "1.64.0" - test_grpc_version = "1.64.0" - jar_location = ( - f"{self.simulated_home}/.library_generation/gapic-generator-java.jar" - ) - # we expect the function to fail because the generator jar is not found in - # its well-known location. To achieve this, we temporarily remove the fake - # generator jar - bash_call(f"rm {jar_location}") - result = self._run_command( - f"download_tools {test_protoc_version} {test_grpc_version} {self.TEST_ARCHITECTURE}" - ) - self.assertEqual(1, result.returncode) - self.assertRegex(result.stdout.decode(), "Please configure your environment") diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh deleted file mode 100755 index 639abd8677..0000000000 --- a/library_generation/test/generate_library_unit_tests.sh +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env bash - -set -xeo pipefail - -# Unit tests against ../utilities.sh -script_dir=$(dirname "$(readlink -f "$0")") -source "${script_dir}"/test_utilities.sh - -# we simulate a properly prepared environment (i.e. generator jar in its -# well-known location). Tests confirming the opposite case will make sure this -# environment is restored -readonly SIMULATED_HOME=$(mktemp -d) -mkdir "${SIMULATED_HOME}/.library_generation" -touch "${SIMULATED_HOME}/.library_generation/gapic-generator-java.jar" -touch "${SIMULATED_HOME}/.library_generation/google-java-format.jar" -HOME="${SIMULATED_HOME}" source "${script_dir}"/../utils/utilities.sh - -# Unit tests -extract_folder_name_test() { - local path="google/cloud/aiplatform/v1/google-cloud-aiplatform-v1-java" - local folder_name - folder_name=$(extract_folder_name "${path}") - assertEquals "google-cloud-aiplatform-v1-java" "${folder_name}" -} - -get_grpc_version_succeed_docker_env_var_test() { - local version_with_docker - local version_without_docker - export DOCKER_GRPC_VERSION="9.9.9" - # get_grpc_version should prioritize DOCKER_GRPC_VERSION - version_with_docker=$(get_grpc_version) - assertEquals "${DOCKER_GRPC_VERSION}" "${version_with_docker}" - unset DOCKER_GRPC_VERSION -} - -get_protoc_version_succeed_docker_env_var_test() { - local version_with_docker - local version_without_docker - export DOCKER_PROTOC_VERSION="9.9.9" - # get_protoc_version should prioritize DOCKER_PROTOC_VERSION - version_with_docker=$(get_protoc_version) - assertEquals "${DOCKER_PROTOC_VERSION}" "${version_with_docker}" - unset DOCKER_PROTOC_VERSION -} - -get_gapic_opts_with_rest_test() { - local proto_path="${script_dir}/resources/gapic_options" - local transport="grpc" - local rest_numeric_enums="true" - local gapic_opts - gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "")" - assertEquals \ - "transport=grpc,rest-numeric-enums,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \ - "${gapic_opts}" -} - -get_gapic_opts_without_rest_test() { - local proto_path="${script_dir}/resources/gapic_options" - local transport="grpc" - local rest_numeric_enums="false" - local gapic_opts - gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "")" - assertEquals \ - "transport=grpc,,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \ - "$gapic_opts" -} - -get_gapic_opts_with_non_default_test() { - local proto_path="${script_dir}/resources/gapic_options" - local transport="grpc" - local rest_numeric_enums="false" - local gapic_opts - gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "${proto_path}/example_gapic.yaml" "${proto_path}/example_grpc_service_config.json" "${proto_path}/example.yaml")" - assertEquals \ - "transport=grpc,,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \ - "$gapic_opts" -} - -remove_grpc_version_test() { - local destination_path="${script_dir}/resources/gapic_options" - cp "${destination_path}/QueryServiceGrpc_copy.java" "${destination_path}/QueryServiceGrpc.java" - remove_grpc_version "${destination_path}" - local res=0 - if ! grep -q 'value = "by gRPC proto compiler",' "${destination_path}/QueryServiceGrpc.java"; then - echo "Error: grpc version is not removed." - res=1 - fi - - assertEquals 0 $((res)) - rm "${destination_path}/QueryServiceGrpc.java" -} - -download_protoc_succeed_with_valid_version_linux_test() { - download_protoc "23.2" "linux-x86_64" - assertFileOrDirectoryExists "protoc-23.2" - rm -rf "protoc-23.2" -} - -download_protoc_succeed_with_valid_version_macos_test() { - download_protoc "23.2" "osx-x86_64" - assertFileOrDirectoryExists "protoc-23.2" - rm -rf "protoc-23.2" "google" -} - -download_protoc_failed_with_invalid_version_linux_test() { - local res=0 - $(download_protoc "22.99" "linux-x86_64") || res=$? - assertEquals 1 $((res)) -} - -download_protoc_failed_with_invalid_arch_test() { - local res=0 - $(download_protoc "23.2" "customized-x86_64") || res=$? - assertEquals 1 $((res)) -} - -download_tools_succeed_with_baked_protoc() { - # This mimics a docker container scenario. - # This test consists of creating an empty /tmp/.../protoc-99.99/bin folder and map - # it to the DOCKER_PROTOC_LOCATION env var (which is treated specially in the - # `download_tools` function). If `DOCKER_PROTOC_VERSION` matches exactly as - # the version passed to `download_protoc`, then we will not download protoc - # but simply have the variable `protoc_path` pointing to DOCKER_PROTOC_LOCATION - # (which we manually created in this test) - export DOCKER_PROTOC_LOCATION=$(mktemp -d) - export DOCKER_PROTOC_VERSION="99.99" - export output_folder=$(get_output_folder) - mkdir "${output_folder}" - local protoc_bin_folder="${DOCKER_PROTOC_LOCATION}/protoc-99.99/bin" - mkdir -p "${protoc_bin_folder}" - - local test_grpc_version="1.64.0" - # we expect download_tools to decide to use DOCKER_PROTOC_LOCATION because - # the protoc version we want to download is the same as DOCKER_PROTOC_VERSION. - # Note that `protoc_bin_folder` is just the expected formatted value that - # download_tools will format using DOCKER_PROTOC_VERSION (via - # download_protoc). - download_tools "99.99" "${test_grpc_version}" "linux-x86_64" - assertEquals "${protoc_bin_folder}" "${protoc_path}" - - rm -rdf "${output_folder}" - unset DOCKER_PROTOC_LOCATION - unset DOCKER_PROTOC_VERSION - unset output_folder - unset protoc_path -} - -download_tools_succeed_with_baked_grpc() { - # This test has the same structure as - # download_tools_succeed_with_baked_protoc, but meant for the grpc plugin. - export DOCKER_GRPC_LOCATION=$(mktemp -d) - export DOCKER_GRPC_VERSION="99.99" - export output_folder=$(get_output_folder) - mkdir "${output_folder}" - - local test_protoc_version="1.64.0" - # we expect download_tools to decide to use DOCKER_GRPC_LOCATION because - # the protoc version we want to download is the same as DOCKER_GRPC_VERSION - download_tools "${test_protoc_version}" "99.99" "linux-x86_64" - assertEquals "${DOCKER_GRPC_LOCATION}" "${grpc_path}" - - rm -rdf "${output_folder}" - unset DOCKER_GRPC_LOCATION - unset DOCKER_GRPC_VERSION - unset output_folder - unset grpc_path -} - -download_grpc_plugin_succeed_with_valid_version_linux_test() { - download_grpc_plugin "1.55.1" "linux-x86_64" - assertFileOrDirectoryExists "protoc-gen-grpc-java-1.55.1-linux-x86_64.exe" - rm "protoc-gen-grpc-java-1.55.1-linux-x86_64.exe" -} - -download_grpc_plugin_succeed_with_valid_version_macos_test() { - download_grpc_plugin "1.55.1" "osx-x86_64" - assertFileOrDirectoryExists "protoc-gen-grpc-java-1.55.1-osx-x86_64.exe" - rm "protoc-gen-grpc-java-1.55.1-osx-x86_64.exe" -} - -download_grpc_plugin_failed_with_invalid_version_linux_test() { - local res=0 - $(download_grpc_plugin "0.99.0" "linux-x86_64") || res=$? - assertEquals 1 $((res)) -} - -download_grpc_plugin_failed_with_invalid_arch_test() { - local res=0 - $(download_grpc_plugin "1.55.1" "customized-x86_64") || res=$? - assertEquals 1 $((res)) -} - -generate_library_failed_with_invalid_generator_version() { - local destination="google-cloud-alloydb-v1-java" - local res=0 - cd "${script_dir}/resources" - bash "${script_dir}"/../generate_library.sh \ - -p google/cloud/alloydb/v1 \ - -d ../"${destination}" \ - --protoc_version 23.2 \ - --grpc_version 1.55.1 \ - --transport grpc+rest \ - --rest_numeric_enums true || res=$? - assertEquals 1 $((res)) - # still need to clean up potential downloaded tooling. - cleanup "${destination}" -} - -generate_library_failed_with_invalid_protoc_version() { - local destination="google-cloud-alloydb-v1-java" - local res=0 - cd "${script_dir}/resources" - bash "${script_dir}"/../generate_library.sh \ - -p google/cloud/alloydb/v1 \ - -d ../"${destination}" \ - --protoc_version 22.99 \ - --grpc_version 1.55.1 \ - --transport grpc+rest \ - --rest_numeric_enums true || res=$? - assertEquals 1 $((res)) - # still need to clean up potential downloaded tooling. - cleanup "${destination}" -} - -generate_library_failed_with_invalid_grpc_version() { - local destination="google-cloud-alloydb-v1-java" - local res=0 - cd "${script_dir}/resources" - bash "${script_dir}"/../generate_library.sh \ - -p google/cloud/alloydb/v1 \ - -d ../output/"${destination}" \ - --grpc_version 0.99.0 \ - --transport grpc+rest \ - --rest_numeric_enums true || res=$? - assertEquals 1 $((res)) - # still need to clean up potential downloaded tooling. - cleanup "${destination}" -} - -copy_directory_if_exists_valid_folder_succeeds() { - local source_folder="${script_dir}/resources" - local destination="${script_dir}/test_destination_folder" - mkdir -p "${destination}" - copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" - n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) - rm -rdf "${destination}" - assertEquals 1 ${n_matching_folders} -} - -copy_directory_if_exists_invalid_folder_does_not_copy() { - local source_folder="${script_dir}/non-existent" - local destination="${script_dir}/test_destination_folder" - mkdir -p "${destination}" - copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" - n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) || res=$? - rm -rdf "${destination}" - assertEquals 0 ${n_matching_folders} -} - -get_proto_path_from_preprocessed_sources_valid_library_succeeds() { - local sources="${script_dir}/resources/proto_path_library" - local proto_path=$(get_proto_path_from_preprocessed_sources "${sources}") - assertEquals "google/cloud/test/v1" ${proto_path} -} - -get_proto_path_from_preprocessed_sources_empty_library_fails() { - local sources=$(mktemp -d) - ( - get_proto_path_from_preprocessed_sources "${sources}" - ) || res=$? - assertEquals 1 ${res} -} - -get_proto_path_from_preprocessed_sources_multiple_proto_dirs_fails() { - local sources="${script_dir}/resources/proto_path_library_multiple_protos" - ( - get_proto_path_from_preprocessed_sources "${sources}" - ) || res=$? - assertEquals 1 ${res} -} - -# Execute tests. -# One line per test. -test_list=( - extract_folder_name_test - get_grpc_version_succeed_docker_env_var_test - get_protoc_version_succeed_docker_env_var_test - get_gapic_opts_with_rest_test - get_gapic_opts_without_rest_test - get_gapic_opts_with_non_default_test - remove_grpc_version_test - download_protoc_succeed_with_valid_version_linux_test - download_protoc_succeed_with_valid_version_macos_test - download_protoc_failed_with_invalid_version_linux_test - download_protoc_failed_with_invalid_arch_test - download_tools_succeed_with_baked_protoc - download_tools_succeed_with_baked_grpc - download_grpc_plugin_succeed_with_valid_version_linux_test - download_grpc_plugin_succeed_with_valid_version_macos_test - download_grpc_plugin_failed_with_invalid_version_linux_test - download_grpc_plugin_failed_with_invalid_arch_test - generate_library_failed_with_invalid_generator_version - generate_library_failed_with_invalid_protoc_version - generate_library_failed_with_invalid_grpc_version - copy_directory_if_exists_valid_folder_succeeds - copy_directory_if_exists_invalid_folder_does_not_copy - get_proto_path_from_preprocessed_sources_valid_library_succeeds - get_proto_path_from_preprocessed_sources_empty_library_fails - get_proto_path_from_preprocessed_sources_multiple_proto_dirs_fails -) - -pushd "${script_dir}" -execute_tests "${test_list[@]}" -popd diff --git a/library_generation/test/generate_pr_description_unit_tests.py b/library_generation/test/generate_pr_description_unit_tests.py deleted file mode 100644 index 9adbe71277..0000000000 --- a/library_generation/test/generate_pr_description_unit_tests.py +++ /dev/null @@ -1,270 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest -from filecmp import cmp - -from library_generation.generate_pr_description import ( - get_repo_level_commit_messages, - generate_pr_descriptions, -) -from library_generation.model.config_change import ( - ConfigChange, - ChangeType, - LibraryChange, -) -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.library_config import LibraryConfig - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "resources", "goldens") - - -class GeneratePrDescriptionTest(unittest.TestCase): - def test_get_commit_messages_current_is_older_raise_exception(self): - # committed on April 1st, 2024 - current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" - # committed on April 2nd, 2024 - baseline_commit = "d5020fff4cbe108bdf506074791c56cff7840bef" - self.assertRaisesRegex( - ValueError, - "newer than", - get_repo_level_commit_messages, - "https://github.com/googleapis/googleapis.git", - current_commit, - baseline_commit, - {}, - True, - [], - ) - - def test_get_commit_messages_with_same_current_and_baseline_returns_empty_message( - self, - ): - # committed on April 1st, 2024 - current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" - baseline_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" - self.assertEqual( - "", - get_repo_level_commit_messages( - "https://github.com/googleapis/googleapis.git", - current_commit, - baseline_commit, - {}, - True, - [], - ), - ) - - def test_generate_pr_description_with_no_change_in_config(self): - commit_sha = "36441693dddaf0ed73951ad3a15c215a332756aa" - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish=commit_sha, - libraries_bom_version="", - # use empty libraries to make sure no qualified commit between - # two commit sha. - libraries=[], - ) - pr_description_path = os.path.join(os.getcwd(), "no_config_change") - generate_pr_descriptions( - config_change=ConfigChange( - change_to_libraries={}, - baseline_config=config, - current_config=config, - ), - description_path=pr_description_path, - ) - self.assertFalse(os.path.isfile(f"{pr_description_path}/pr_description.txt")) - - def test_generate_pr_description_does_not_create_pr_description_without_qualified_commit( - self, - ): - # committed on May 22nd, 2024 - old_commit_sha = "30717c0b0c9966906880703208a4c820411565c4" - # committed on May 23rd, 2024 - new_commit_sha = "eeed69d446a90eb4a4a2d1762c49d637075390c1" - pr_description_path = os.path.join(os.getcwd(), "no_qualified_commit") - generate_pr_descriptions( - config_change=ConfigChange( - change_to_libraries={}, - baseline_config=GenerationConfig( - gapic_generator_version="", - googleapis_commitish=old_commit_sha, - # use empty libraries to make sure no qualified commit between - # two commit sha. - libraries=[], - ), - current_config=GenerationConfig( - gapic_generator_version="", - googleapis_commitish=new_commit_sha, - # use empty libraries to make sure no qualified commit between - # two commit sha. - libraries=[], - ), - ), - description_path=pr_description_path, - ) - self.assertFalse(os.path.isfile(f"{pr_description_path}/pr_description.txt")) - - def test_generate_pr_description_with_combined_message( - self, - ): - # no other commits between these two commits. - baseline_commit_sha = "3b6f144d47b0a1d2115ab2445ec06e80cc324a44" - documentai_commit_sha = "0cea7170404bec3d994f43db4fa292f5034cbe9a" - cwd = os.getcwd() - library = LibraryConfig( - api_shortname="documentai", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[GapicConfig(proto_path="google/cloud/documentai/v1")], - ) - generate_pr_descriptions( - config_change=ConfigChange( - change_to_libraries={ - ChangeType.REPO_LEVEL_CHANGE: [ - LibraryChange( - changed_param="gapic_generator_version", - current_value="1.2.3", - ), - LibraryChange( - changed_param="libraries_bom_version", current_value="2.3.4" - ), - ], - ChangeType.GOOGLEAPIS_COMMIT: [], - }, - baseline_config=GenerationConfig( - gapic_generator_version="", - googleapis_commitish=baseline_commit_sha, - libraries=[library], - ), - current_config=GenerationConfig( - gapic_generator_version="1.2.3", - googleapis_commitish=documentai_commit_sha, - libraries_bom_version="2.3.4", - libraries=[library], - ), - ), - description_path=cwd, - ) - self.assertTrue(os.path.isfile(f"{cwd}/pr_description.txt")) - self.assertTrue( - cmp( - f"{resources_dir}/pr_description-golden.txt", - f"{cwd}/pr_description.txt", - ), - "The generated PR description does not match the expected golden file", - ) - os.remove(f"{cwd}/pr_description.txt") - - def test_generate_pr_description_with_repo_level_change_without_qualified_commit( - self, - ): - # no other commits between these two commits. - baseline_commit_sha = "3b6f144d47b0a1d2115ab2445ec06e80cc324a44" - current_commit_sha = "0cea7170404bec3d994f43db4fa292f5034cbe9a" - cwd = os.getcwd() - library = LibraryConfig( - api_shortname="example_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[GapicConfig(proto_path="google/example/v1")], - ) - generate_pr_descriptions( - config_change=ConfigChange( - change_to_libraries={ - ChangeType.REPO_LEVEL_CHANGE: [ - LibraryChange( - changed_param="gapic_generator_version", - current_value="1.2.3", - ), - LibraryChange( - changed_param="libraries_bom_version", current_value="2.3.4" - ), - ], - ChangeType.GOOGLEAPIS_COMMIT: [], - }, - baseline_config=GenerationConfig( - gapic_generator_version="", - googleapis_commitish=baseline_commit_sha, - libraries=[library], - ), - current_config=GenerationConfig( - gapic_generator_version="1.2.3", - googleapis_commitish=current_commit_sha, - libraries_bom_version="2.3.4", - libraries=[library], - ), - ), - description_path=cwd, - ) - self.assertTrue(os.path.isfile(f"{cwd}/pr_description.txt")) - self.assertTrue( - cmp( - f"{resources_dir}/repo_level_and_no_qualified_commit_pr_description-golden.txt", - f"{cwd}/pr_description.txt", - ), - "The generated PR description does not match the expected golden file", - ) - os.remove(f"{cwd}/pr_description.txt") - - def test_generate_pr_description_create_description_with_only_repo_level_change( - self, - ): - commit_sha = "3b6f144d47b0a1d2115ab2445ec06e80cc324a44" - cwd = os.getcwd() - library = LibraryConfig( - api_shortname="documentai", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[GapicConfig(proto_path="google/cloud/documentai/v1")], - ) - generate_pr_descriptions( - config_change=ConfigChange( - change_to_libraries={ - ChangeType.REPO_LEVEL_CHANGE: [ - LibraryChange( - changed_param="gapic_generator_version", - current_value="1.2.3", - ) - ], - ChangeType.GOOGLEAPIS_COMMIT: [], - }, - baseline_config=GenerationConfig( - gapic_generator_version="1.2.2", - googleapis_commitish=commit_sha, - libraries=[library], - ), - current_config=GenerationConfig( - gapic_generator_version="1.2.3", - googleapis_commitish=commit_sha, - libraries=[library], - ), - ), - description_path=cwd, - ) - self.assertTrue(os.path.isfile(f"{cwd}/pr_description.txt")) - self.assertTrue( - cmp( - f"{resources_dir}/repo_level_only_pr_description-golden.txt", - f"{cwd}/pr_description.txt", - ), - "The generated PR description does not match the expected golden file", - ) - os.remove(f"{cwd}/pr_description.txt") diff --git a/library_generation/test/generate_repo_unit_tests.py b/library_generation/test/generate_repo_unit_tests.py deleted file mode 100644 index 470f0a4b18..0000000000 --- a/library_generation/test/generate_repo_unit_tests.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from library_generation.generate_repo import get_target_libraries -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.library_config import LibraryConfig - - -class GenerateRepoTest(unittest.TestCase): - def test_get_target_library_returns_selected_libraries(self): - one_library = GenerateRepoTest.__get_an_empty_library_config() - one_library.api_shortname = "one_library" - another_library = GenerateRepoTest.__get_an_empty_library_config() - another_library.api_shortname = "another_library" - config = GenerateRepoTest.__get_an_empty_generation_config() - config.libraries.extend([one_library, another_library]) - target_libraries = get_target_libraries(config, ["another_library"]) - self.assertEqual([another_library], target_libraries) - - def test_get_target_library_given_null_returns_all_libraries(self): - one_library = GenerateRepoTest.__get_an_empty_library_config() - one_library.api_shortname = "one_library" - another_library = GenerateRepoTest.__get_an_empty_library_config() - another_library.api_shortname = "another_library" - config = GenerateRepoTest.__get_an_empty_generation_config() - config.libraries.extend([one_library, another_library]) - target_libraries = get_target_libraries(config) - self.assertEqual([one_library, another_library], target_libraries) - - def test_get_target_library_given_an_non_existent_library_returns_only_existing_libraries( - self, - ): - one_library = GenerateRepoTest.__get_an_empty_library_config() - one_library.api_shortname = "one_library" - another_library = GenerateRepoTest.__get_an_empty_library_config() - another_library.api_shortname = "another_library" - config = GenerateRepoTest.__get_an_empty_generation_config() - config.libraries.extend([one_library, another_library]) - target_libraries = get_target_libraries( - config, ["one_library", "another_library", "non_existent_library"] - ) - self.assertEqual([one_library, another_library], target_libraries) - - @staticmethod - def __get_an_empty_generation_config() -> GenerationConfig: - return GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[], - ) - - @staticmethod - def __get_an_empty_library_config() -> LibraryConfig: - return LibraryConfig( - api_shortname="", - name_pretty="", - api_description="", - product_documentation="", - gapic_configs=[], - ) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py deleted file mode 100644 index 576fb21ede..0000000000 --- a/library_generation/test/integration_tests.py +++ /dev/null @@ -1,394 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from click.testing import CliRunner -import difflib -import json -import tempfile -from filecmp import cmp -from filecmp import dircmp -from git import Repo -import os -import shutil -import subprocess -import unittest -from distutils.dir_util import copy_tree -from distutils.file_util import copy_file -from pathlib import Path -from library_generation.cli.generate_release_note import ( - generate as generate_pr_description, -) -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.generation_config import from_yaml -from library_generation.test.compare_poms import compare_xml -from library_generation.utils.utilities import sh_util as shell_call - - -script_dir = os.path.dirname(os.path.realpath(__file__)) -config_dir = os.path.join(script_dir, "resources", "integration") -golden_dir = os.path.join(config_dir, "golden") -generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") -repo_root_dir = os.path.join(script_dir, "..", "..") -build_file = os.path.join( - repo_root_dir, ".cloudbuild", "library_generation", "library_generation.Dockerfile" -) -image_tag = "test-image:latest" -repo_prefix = "https://github.com/googleapis" -output_dir = shell_call("get_output_folder") -# this map tells which branch of each repo should we use for our diff tests -commitish_map = { - "google-cloud-java": "chore/test-hermetic-build", - "java-bigtable": "chore/test-hermetic-build", -} -baseline_config_name = "baseline_generation_config.yaml" -current_config_name = "current_generation_config.yaml" -googleapis_commitish = "113a378d5aad5018876ec0a8cbfd4d6a4f746809" -# This variable is used to override the jar created by building the image -# with our own downloaded jar in order to lock the integration test to use -# a constant version specified in -# library_generation/test/resources/integration/test_generator_coordinates -# This allows us to decouple the generation workflow testing with what the -# generator jar will actually generate. -# See library_generation/DEVELOPMENT.md ("The Hermetic Build's -# well-known folder"). -WELL_KNOWN_GENERATOR_JAR_FILENAME = "gapic-generator-java.jar" - - -class IntegrationTest(unittest.TestCase): - @classmethod - def setUpClass(cls) -> None: - cls.__download_generator_jar(coordinates_file=generator_jar_coordinates_file) - cls.__build_image(docker_file=build_file, cwd=repo_root_dir) - - @classmethod - def setUp(cls) -> None: - cls.__remove_generated_files() - os.makedirs(f"{golden_dir}", exist_ok=True) - - def test_entry_point_running_in_container(self): - api_definitions_path = self.__copy_api_definition(googleapis_commitish) - config_files = self.__get_config_files(config_dir) - for repo, config_file in config_files: - config = from_yaml(config_file) - repo_location = f"{output_dir}/{repo}" - config_location = f"{golden_dir}/../{repo}" - # 1. pull repository - repo_dest = self.__pull_repo_to( - Path(repo_location), repo, commitish_map[repo] - ) - # 2. prepare golden files - library_names = self.__get_library_names_from_config(config) - self.__prepare_golden_files( - config=config, library_names=library_names, repo_dest=repo_dest - ) - # 3. run entry_point.py in docker container - self.__run_entry_point_in_docker_container( - repo_location=repo_location, - config_location=config_location, - baseline_config=baseline_config_name, - current_config=current_config_name, - api_definition=api_definitions_path, - ) - # 4. generate pr description - # noinspection PyTypeChecker - result = CliRunner().invoke( - generate_pr_description, - [ - f"--baseline-generation-config-path={config_location}/{baseline_config_name}", - f"--current-generation-config-path={config_location}/{current_config_name}", - f"--repository-path={repo_location}", - ], - ) - self.assertEqual(0, result.exit_code) - # 5. compare generation result with golden files - print( - "Generation finished successfully. " - "Will now compare differences between generated and existing " - "libraries" - ) - for library_name in library_names: - actual_library = ( - f"{repo_dest}/{library_name}" if config.is_monorepo() else repo_dest - ) - print("*" * 50) - print(f"Checking for differences in '{library_name}'.") - print(f" The expected library is in {golden_dir}/{library_name}.") - print(f" The actual library is in {actual_library}. ") - compare_result = dircmp( - f"{golden_dir}/{library_name}", - actual_library, - ignore=[".repo-metadata.json"], - ) - diff_files = [] - golden_only = [] - generated_only = [] - # compare source code - self.__recursive_diff_files( - compare_result, diff_files, golden_only, generated_only - ) - - # print all found differences for inspection - print_file = lambda f: print(f" - {f}") - if len(diff_files) > 0: - print(" Some files (found in both folders) are differing:") - for diff_file in diff_files: - print(f"Difference in {diff_file}:") - with open( - f"{golden_dir}/{library_name}/{diff_file}" - ) as expected_file: - with open(f"{actual_library}/{diff_file}") as actual_file: - [ - print(line) - for line in difflib.unified_diff( - expected_file.readlines(), - actual_file.readlines(), - ) - ] - if len(golden_only) > 0: - print(" There were files found only in the golden dir:") - [print_file(f) for f in golden_only] - if len(generated_only) > 0: - print(" There were files found only in the generated dir:") - [print_file(f) for f in generated_only] - - self.assertTrue(len(golden_only) == 0) - self.assertTrue(len(generated_only) == 0) - self.assertTrue(len(diff_files) == 0) - - print(f" No differences found in {library_name}") - # compare .repo-metadata.json - self.assertTrue( - self.__compare_json_files( - f"{golden_dir}/{library_name}/.repo-metadata.json", - f"{actual_library}/.repo-metadata.json", - ), - msg=f" The generated {library_name}/.repo-metadata.json is different from golden.", - ) - print(" .repo-metadata.json comparison succeed.") - - if not config.is_monorepo(): - continue - - # compare gapic-libraries-bom/pom.xml and pom.xml - self.assertFalse( - compare_xml( - f"{golden_dir}/gapic-libraries-bom/pom.xml", - f"{repo_dest}/gapic-libraries-bom/pom.xml", - False, - ) - ) - print(" gapic-libraries-bom/pom.xml comparison succeed.") - self.assertFalse( - compare_xml( - f"{golden_dir}/pom.xml", - f"{repo_dest}/pom.xml", - False, - ) - ) - print(" pom.xml comparison succeed.") - # compare PR description - description_file = f"{output_dir}/{repo}/pr_description.txt" - self.assertTrue( - cmp( - f"{config_dir}/{repo}/pr-description-golden.txt", - f"{description_file}", - ), - "The generated PR description does not match the expected golden file", - ) - print(" PR description comparison succeed.") - self.__remove_generated_files() - shutil.rmtree(api_definitions_path) - - @classmethod - def __copy_api_definition(cls, committish: str) -> str: - repo_dest = cls.__pull_repo_to( - dest=tempfile.mkdtemp(), repo="googleapis", committish=committish - ) - api_temp_dir = tempfile.mkdtemp() - print(f"Copying api definition to {api_temp_dir}...") - shutil.copytree( - f"{repo_dest}/google", f"{api_temp_dir}/google", dirs_exist_ok=True - ) - shutil.copytree( - f"{repo_dest}/grafeas", f"{api_temp_dir}/grafeas", dirs_exist_ok=True - ) - shutil.rmtree(repo_dest) - return api_temp_dir - - @classmethod - def __build_image(cls, docker_file: str, cwd: str): - # we build the docker image without removing intermediate containers, so - # we can re-test more quickly - subprocess.check_call( - ["docker", "build", "-f", docker_file, "-t", image_tag, "."], - cwd=cwd, - ) - - @classmethod - def __download_generator_jar(cls, coordinates_file: str) -> None: - """ - Downloads the jar at the version specified in the - coordinates file - :param coordinates_file: path to the file containing the coordinates - """ - with open(coordinates_file, "r") as coordinates_file_handle: - # make this var available in the function scope - # nonlocal coordinates - coordinates = coordinates_file_handle.read() - # download the jar - subprocess.check_call( - [ - "mvn", - "dependency:copy", - f"-Dartifact={coordinates}", - f"-DoutputDirectory={config_dir}", - ] - ) - - # compute the filename of the downloaded jar - split_coordinates = coordinates.split(":") - artifact_id = split_coordinates[1] - version = split_coordinates[2] - jar_filename = f"{artifact_id}-{version}.jar" - - # rename the jar to its well-known filename defined at the top of this - # script file - source_jar_path = os.path.join(config_dir, jar_filename) - destination_jar_path = os.path.join( - config_dir, WELL_KNOWN_GENERATOR_JAR_FILENAME - ) - shutil.move(source_jar_path, destination_jar_path) - - @classmethod - def __remove_generated_files(cls): - shutil.rmtree(f"{output_dir}", ignore_errors=True) - if os.path.isdir(f"{golden_dir}"): - shutil.rmtree(f"{golden_dir}") - - @classmethod - def __pull_repo_to(cls, dest: Path, repo: str, committish: str) -> str: - shutil.rmtree(dest, ignore_errors=True) - repo_url = f"{repo_prefix}/{repo}" - print(f"Cloning repository {repo_url}") - repo = Repo.clone_from(repo_url, dest) - repo.git.checkout(committish) - return str(dest) - - @classmethod - def __get_library_names_from_config(cls, config: GenerationConfig) -> list[str]: - library_names = [] - for library in config.libraries: - library_names.append(f"java-{library.get_library_name()}") - - return library_names - - @classmethod - def __prepare_golden_files( - cls, config: GenerationConfig, library_names: list[str], repo_dest: str - ): - for library_name in library_names: - if config.is_monorepo(): - copy_tree(f"{repo_dest}/{library_name}", f"{golden_dir}/{library_name}") - copy_tree( - f"{repo_dest}/gapic-libraries-bom", - f"{golden_dir}/gapic-libraries-bom", - ) - copy_file(f"{repo_dest}/pom.xml", golden_dir) - else: - copy_tree(f"{repo_dest}", f"{golden_dir}/{library_name}") - - @classmethod - def __run_entry_point_in_docker_container( - cls, - repo_location: str, - config_location: str, - baseline_config: str, - current_config: str, - api_definition: str, - ): - # we use the calling user to prevent the mapped volumes from changing - # owners - user_id = shell_call("id -u") - group_id = shell_call("id -g") - subprocess.check_call( - [ - "docker", - "run", - "-u", - f"{user_id}:{group_id}", - "--rm", - "-v", - f"{repo_location}:/workspace/repo", - "-v", - f"{config_location}:/workspace/config", - "-v", - f"{api_definition}:/workspace/api", - "-v", - f"{config_dir}/{WELL_KNOWN_GENERATOR_JAR_FILENAME}:/home/.library_generation/{WELL_KNOWN_GENERATOR_JAR_FILENAME}", - "-w", - "/workspace/repo", - image_tag, - f"--baseline-generation-config-path=/workspace/config/{baseline_config}", - f"--current-generation-config-path=/workspace/config/{current_config}", - f"--api-definitions-path=/workspace/api", - ], - ) - - @classmethod - def __get_config_files(cls, path: str) -> list[tuple[str, str]]: - config_files = [] - for sub_dir in Path(path).resolve().iterdir(): - if sub_dir.is_file(): - continue - repo = sub_dir.name - if repo in ["golden", "java-bigtable"]: - continue - config = f"{sub_dir}/{current_config_name}" - config_files.append((repo, config)) - return config_files - - @classmethod - def __compare_json_files(cls, expected: str, actual: str) -> bool: - return cls.__load_json_to_sorted_list( - expected - ) == cls.__load_json_to_sorted_list(actual) - - @classmethod - def __load_json_to_sorted_list(cls, path: str) -> list[tuple]: - with open(path) as f: - data = json.load(f) - res = [(key, value) for key, value in data.items()] - - return sorted(res, key=lambda x: x[0]) - - @classmethod - def __recursive_diff_files( - cls, - dcmp: dircmp, - diff_files: list[str], - left_only: list[str], - right_only: list[str], - dirname: str = "", - ): - """ - Recursively compares two subdirectories. The found differences are - passed to three expected list references. - """ - append_dirname = lambda d: dirname + d - diff_files.extend(map(append_dirname, dcmp.diff_files)) - left_only.extend(map(append_dirname, dcmp.left_only)) - right_only.extend(map(append_dirname, dcmp.right_only)) - for sub_dirname, sub_dcmp in dcmp.subdirs.items(): - cls.__recursive_diff_files( - sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" - ) diff --git a/library_generation/test/model/__init__.py b/library_generation/test/model/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/model/config_change_unit_tests.py b/library_generation/test/model/config_change_unit_tests.py deleted file mode 100644 index 147753ec99..0000000000 --- a/library_generation/test/model/config_change_unit_tests.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from library_generation.model.config_change import ChangeType -from library_generation.model.config_change import ConfigChange -from library_generation.model.config_change import LibraryChange -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.library_config import LibraryConfig - - -class ConfigChangeTest(unittest.TestCase): - def test_get_changed_libraries_with_repo_level_change_returns_all_libraries_changed( - self, - ): - config_change = ConfigChange( - change_to_libraries={ - ChangeType.REPO_LEVEL_CHANGE: [], - # add a library level change to verify this type of change has - # no impact on the result. - ChangeType.LIBRARY_LEVEL_CHANGE: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="test-library", - ) - ], - }, - baseline_config=ConfigChangeTest.__get_a_gen_config(), - current_config=ConfigChangeTest.__get_a_gen_config(), - ) - self.assertEqual( - ConfigChange.ALL_LIBRARIES_CHANGED, - config_change.get_changed_libraries(), - ) - - def test_get_changed_libraries_with_library_level_change_returns_list(self): - config_change = ConfigChange( - change_to_libraries={ - ChangeType.LIBRARY_LEVEL_CHANGE: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="a-library", - ), - LibraryChange( - changed_param="test", - current_value="test", - library_name="another-library", - ), - ], - ChangeType.LIBRARIES_ADDITION: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="new-library", - ), - ], - ChangeType.GAPIC_ADDITION: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="library-with-new-version", - ), - ], - }, - baseline_config=ConfigChangeTest.__get_a_gen_config(), - current_config=ConfigChangeTest.__get_a_gen_config(), - ) - self.assertEqual( - ["a-library", "another-library", "library-with-new-version", "new-library"], - config_change.get_changed_libraries(), - ) - - def test_get_changed_libraries_with_duplicated_library_name_returns_unique_name( - self, - ): - config_change = ConfigChange( - change_to_libraries={ - ChangeType.LIBRARY_LEVEL_CHANGE: [ - LibraryChange( - changed_param="a-param", - current_value="new_test", - library_name="a-library", - ), - LibraryChange( - changed_param="another-param", - current_value="new_value", - library_name="a-library", - ), - ], - }, - baseline_config=ConfigChangeTest.__get_a_gen_config(), - current_config=ConfigChangeTest.__get_a_gen_config(), - ) - self.assertEqual( - ["a-library"], - config_change.get_changed_libraries(), - ) - - def test_get_changed_libraries_with_mix_changes_returns_list(self): - baseline_commit = "277145d108819fa30fbed3a7cbbb50f91eb6155e" - latest_commit = "8984ddb508dea0e673b724c58338e810b1d8aee3" - config_change = ConfigChange( - change_to_libraries={ - ChangeType.GOOGLEAPIS_COMMIT: [], - ChangeType.LIBRARY_LEVEL_CHANGE: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="a-library", - ) - ], - ChangeType.LIBRARIES_ADDITION: [ - LibraryChange( - changed_param="test", - current_value="test", - library_name="new-library", - ), - ], - }, - baseline_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=baseline_commit - ), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=latest_commit, - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="gke-backup", - gapic_configs=[ - GapicConfig(proto_path="google/cloud/gkebackup/v1") - ], - ), - ], - ), - ) - - self.assertEqual( - ["a-library", "gke-backup", "new-library"], - sorted(config_change.get_changed_libraries()), - ) - - def test_get_qualified_commits_success(self): - baseline_commit = "277145d108819fa30fbed3a7cbbb50f91eb6155e" - latest_commit = "8984ddb508dea0e673b724c58338e810b1d8aee3" - gke_backup_commit = "b8691edb3f1d3c1583aa9cd89240eb359eebe9c7" - aiplatform_commit = "b82095baef02e525bee7bb1c48911c33b66acdf0" - network_management_commit = "efad09c9f0d46ae0786d810a88024363e06c6ca3" - config_change = ConfigChange( - change_to_libraries={}, - baseline_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=baseline_commit - ), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=latest_commit, - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="gke-backup", - gapic_configs=[ - GapicConfig(proto_path="google/cloud/gkebackup/v1") - ], - ), - ConfigChangeTest.__get_a_library_config( - library_name="aiplatform", - gapic_configs=[ - GapicConfig(proto_path="google/cloud/aiplatform/v1beta1") - ], - ), - ConfigChangeTest.__get_a_library_config( - library_name="network-management", - gapic_configs=[ - GapicConfig(proto_path="google/cloud/networkmanagement/v1"), - GapicConfig( - proto_path="google/cloud/networkmanagement/v1beta1" - ), - ], - ), - ], - ), - ) - qualified_commits = config_change.get_qualified_commits() - self.assertEqual(3, len(qualified_commits)) - self.assertEqual({"gke-backup"}, qualified_commits[0].libraries) - self.assertEqual( - gke_backup_commit, - qualified_commits[0].commit.hexsha, - ) - self.assertEqual({"aiplatform"}, qualified_commits[1].libraries) - self.assertEqual( - aiplatform_commit, - qualified_commits[1].commit.hexsha, - ) - self.assertEqual({"network-management"}, qualified_commits[2].libraries) - self.assertEqual( - network_management_commit, - qualified_commits[2].commit.hexsha, - ) - - def test_get_qualified_commits_rest_numeric_enum_change_returns_a_qualified_commit( - self, - ): - baseline_commit = "45694d2bad602c52170096072d325aa644d550e5" - latest_commit = "758f0d1217d9c7fe398aa5efb1057ce4b6409e55" - config_change = ConfigChange( - change_to_libraries={}, - baseline_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=baseline_commit - ), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=latest_commit, - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="container", - gapic_configs=[GapicConfig(proto_path="google/container/v1")], - ) - ], - ), - ) - # one commit between latest_commit and baseline_commit which only - # changed BUILD.bazel. - # this commit changed `rest_numeric_enums`. - self.assertTrue(len(config_change.get_qualified_commits()) == 1) - - def test_get_qualified_commits_irrelevant_build_field_change_returns_empty_list( - self, - ): - baseline_commit = "bdda0174f68a738518ec311e05e6fd9bbe19cd78" - latest_commit = "c9a5050ef225b0011603e1109cf53ab1de0a8e53" - config_change = ConfigChange( - change_to_libraries={}, - baseline_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=baseline_commit - ), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=latest_commit, - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="chat", - gapic_configs=[GapicConfig(proto_path="google/chat/v1")], - ) - ], - ), - ) - # one commit between latest_commit and baseline_commit which only - # changed BUILD.bazel. - # this commit didn't change fields used in library generation. - self.assertTrue(len(config_change.get_qualified_commits()) == 0) - - def test_get_qualified_commits_add_build_file_returns_a_qualified_commit(self): - baseline_commit = "d007ca1b3cc820651530d44d5388533047ae1414" - latest_commit = "05d889e7dfe087fc2ddc9de9579f01d4e1c2f35e" - config_change = ConfigChange( - change_to_libraries={}, - baseline_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=baseline_commit - ), - current_config=ConfigChangeTest.__get_a_gen_config( - googleapis_commitish=latest_commit, - libraries=[ - ConfigChangeTest.__get_a_library_config( - library_name="cloudcontrolspartner", - gapic_configs=[ - GapicConfig( - proto_path="google/cloud/cloudcontrolspartner/v1" - ) - ], - ) - ], - ), - ) - # one commit between latest_commit and baseline_commit which added - # google/cloud/cloudcontrolspartner/v1. - self.assertTrue(len(config_change.get_qualified_commits()) == 1) - - @staticmethod - def __get_a_gen_config( - googleapis_commitish="", libraries: list[LibraryConfig] = None - ) -> GenerationConfig: - if libraries is None: - libraries = [] - return GenerationConfig( - gapic_generator_version="", - googleapis_commitish=googleapis_commitish, - grpc_version="", - protoc_version="", - libraries=libraries, - ) - - @staticmethod - def __get_a_library_config( - library_name: str, gapic_configs: list[GapicConfig] = None - ) -> LibraryConfig: - if gapic_configs is None: - gapic_configs = [] - return LibraryConfig( - api_shortname="existing_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=gapic_configs, - library_name=library_name, - ) diff --git a/library_generation/test/model/gapic_config_unit_tests.py b/library_generation/test/model/gapic_config_unit_tests.py deleted file mode 100644 index 64d8556648..0000000000 --- a/library_generation/test/model/gapic_config_unit_tests.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from library_generation.model.gapic_config import GapicConfig - - -class GapicConfigTest(unittest.TestCase): - def test_get_version_returns_none(self): - self.assertIsNone(GapicConfig(proto_path="example/dir1/dir2").get_version()) - - def test_get_version_returns_non_stable_version(self): - self.assertEqual( - "v2p2beta1", - GapicConfig(proto_path="example/dir1/dir2/v2p2beta1").get_version(), - ) - self.assertEqual( - "v2beta1", - GapicConfig(proto_path="example/dir1/dir2/v2beta1").get_version(), - ) - - def test_get_version_returns_stable_version(self): - self.assertEqual( - "v20", - GapicConfig(proto_path="example/dir1/dir2/v20").get_version(), - ) - - def test_is_stable_with_no_version_returns_false(self): - self.assertFalse( - GapicConfig(proto_path="example/dir1/dir2/non_version").is_stable(), - ) - - def test_is_stable_with_non_stable_version_returns_false(self): - self.assertFalse( - GapicConfig(proto_path="example/dir1/dir2/v20alpha").is_stable(), - ) - self.assertFalse( - GapicConfig(proto_path="example/dir1/dir2/v20beta2").is_stable(), - ) - - def test_is_stable_with_stable_version_returns_true(self): - self.assertTrue( - GapicConfig(proto_path="example/dir1/dir2/v30").is_stable(), - ) - - def test_compare_configs_without_a_version(self): - config_len_3 = GapicConfig(proto_path="example/dir1/dir2") - config_len_4 = GapicConfig(proto_path="example/dir1/dir2/dir3") - self.assertLess( - config_len_3, - config_len_4, - "config_len_3 should be smaller since it has a lower depth.", - ) - - def test_compare_configs_only_one_has_a_stable_version(self): - versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v1") - non_versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3") - self.assertLess( - versioned_config, - non_versioned_config, - "versioned_config should be smaller since it has a version.", - ) - - def test_compare_configs_only_one_has_a_non_stable_version(self): - non_stable_versioned_config = GapicConfig( - proto_path="example/dir1/dir2/dir3/dir4/v1beta" - ) - non_versioned_config = GapicConfig(proto_path="example/dir1/dir2/dir3") - self.assertLess( - non_stable_versioned_config, - non_versioned_config, - "non_stable_versioned_config should be smaller since it has a version.", - ) - - def test_compare_configs_one_has_non_stable_and_one_has_stable_version(self): - stable_versioned_config = GapicConfig( - proto_path="example/dir1/dir2/dir3/dir4/v1" - ) - non_stable_versioned_config = GapicConfig(proto_path="example/dir1/dir2/v2beta") - self.assertLess( - stable_versioned_config, - non_stable_versioned_config, - "stable_versioned_config should be smaller since it has a stable version.", - ) - - def test_compare_configs_two_have_non_stable_version(self): - v3p2beta = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v3p2beta") - v2p4beta = GapicConfig(proto_path="example/dir1/dir2/v2p4beta") - self.assertLess( - v3p2beta, - v2p4beta, - "v3p2beta should be smaller since it has a higher version.", - ) - - def test_compare_configs_two_have_stable_version_different_depth(self): - v3 = GapicConfig(proto_path="example/dir1/dir2/v3") - v4 = GapicConfig(proto_path="example/dir1/dir2/dir3/dir4/v4") - self.assertLess( - v3, - v4, - "v3 should be smaller since it has lower depth", - ) - - def test_compare_configs_two_have_stable_version_same_depth(self): - v4 = GapicConfig(proto_path="example/dir1/dir2/v4") - v3 = GapicConfig(proto_path="example/dir1/dir2/v3") - self.assertLess( - v4, - v3, - "v4 should be smaller since it has a higher version", - ) diff --git a/library_generation/test/model/gapic_inputs_unit_tests.py b/library_generation/test/model/gapic_inputs_unit_tests.py deleted file mode 100644 index 210d321591..0000000000 --- a/library_generation/test/model/gapic_inputs_unit_tests.py +++ /dev/null @@ -1,132 +0,0 @@ -import os -import unittest -from pathlib import Path - -from parameterized import parameterized -from library_generation.model.gapic_inputs import parse - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") -build_file = Path(os.path.join(resources_dir, "misc")).resolve() - - -class UtilitiesTest(unittest.TestCase): - @parameterized.expand( - [ - ("BUILD_no_additional_protos.bazel", " "), - ("BUILD_common_resources.bazel", " google/cloud/common_resources.proto"), - ("BUILD_comment_common_resources.bazel", " "), - ("BUILD_locations.bazel", " google/cloud/location/locations.proto"), - ("BUILD_comment_locations.bazel", " "), - ("BUILD_iam_policy.bazel", " google/iam/v1/iam_policy.proto"), - ("BUILD_comment_iam_policy.bazel", " "), - ( - "BUILD_iam_locations.bazel", - " google/cloud/location/locations.proto google/iam/v1/iam_policy.proto", - ), - ] - ) - def test_gapic_inputs_parse_additional_protos(self, build_name, expected): - parsed = parse(build_file, "", build_name) - self.assertEqual( - expected, - parsed.additional_protos, - ) - - def test_gapic_inputs_parse_grpc_only_succeeds(self): - parsed = parse(build_file, "", "BUILD_grpc.bazel") - self.assertEqual("grpc", parsed.transport) - - def test_gapic_inputs_parse_grpc_rest_succeeds(self): - parsed = parse(build_file, "", "BUILD_grpc_rest.bazel") - self.assertEqual("grpc+rest", parsed.transport) - - def test_gapic_inputs_parse_rest_succeeds(self): - parsed = parse(build_file, "", "BUILD_rest.bazel") - self.assertEqual("rest", parsed.transport) - - def test_gapic_inputs_parse_empty_include_samples_succeeds(self): - parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel") - self.assertEqual("false", parsed.include_samples) - - def test_gapic_inputs_parse_include_samples_false_succeeds(self): - parsed = parse(build_file, "", "BUILD_include_samples_false.bazel") - self.assertEqual("false", parsed.include_samples) - - def test_gapic_inputs_parse_include_samples_true_succeeds(self): - parsed = parse(build_file, "", "BUILD_include_samples_true.bazel") - self.assertEqual("true", parsed.include_samples) - - def test_gapic_inputs_parse_empty_rest_numeric_enums_succeeds(self): - parsed = parse(build_file, "", "BUILD_rest_numeric_enums_empty.bazel") - self.assertEqual("false", parsed.rest_numeric_enum) - - def test_gapic_inputs_parse_rest_numeric_enums_false_succeeds(self): - parsed = parse(build_file, "", "BUILD_rest_numeric_enums_false.bazel") - self.assertEqual("false", parsed.rest_numeric_enum) - - def test_gapic_inputs_parse_rest_numeric_enums_true_succeeds(self): - parsed = parse(build_file, "", "BUILD_rest_numeric_enums_true.bazel") - self.assertEqual("true", parsed.rest_numeric_enum) - - def test_gapic_inputs_parse_no_gapic_library_returns_proto_only_true(self): - # include_samples_empty only has a gradle assembly rule - parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel") - self.assertEqual("true", parsed.proto_only) - - def test_gapic_inputs_parse_with_gapic_library_returns_proto_only_false(self): - # rest.bazel has a java_gapic_library rule - parsed = parse(build_file, "", "BUILD_rest.bazel") - self.assertEqual("false", parsed.proto_only) - - def test_gapic_inputs_parse_gapic_yaml_succeeds(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_gapic_yaml.bazel") - self.assertEqual("test/versioned/path/test_gapic_yaml.yaml", parsed.gapic_yaml) - - def test_gapic_inputs_parse_no_gapic_yaml_returns_empty_string(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_no_gapic_yaml.bazel") - self.assertEqual("", parsed.gapic_yaml) - - def test_gapic_inputs_parse_service_config_succeeds(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_service_config.bazel") - self.assertEqual( - "test/versioned/path/test_service_config.json", parsed.service_config - ) - - def test_gapic_inputs_parse_service_yaml_relative_target(self): - parsed = parse( - build_file, - "google/cloud/compute/v1", - "BUILD_service_config_relative_target.bazel", - ) - self.assertEqual( - "google/cloud/compute/v1/compute_grpc_service_config.json", - parsed.service_config, - ) - - def test_gapic_inputs_parse_no_service_config_returns_empty_string(self): - parsed = parse( - build_file, "test/versioned/path", "BUILD_no_service_config.bazel" - ) - self.assertEqual("", parsed.service_config) - - def test_gapic_inputs_parse_service_yaml_succeeds(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_service_yaml.bazel") - self.assertEqual( - "test/versioned/path/test_service_yaml.yaml", parsed.service_yaml - ) - - def test_gapic_inputs_parse_service_yaml_absolute_target(self): - parsed = parse(build_file, "", "BUILD_service_yaml_absolute_target.bazel") - self.assertEqual( - "google/cloud/videointelligence/videointelligence_v1p3beta1.yaml", - parsed.service_yaml, - ) - - def test_gapic_inputs_parse_no_service_yaml_returns_empty_string(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_no_service_yaml.bazel") - self.assertEqual("", parsed.service_yaml) - - def test_gapic_inputs_parse_proto_only_returns_grpc(self): - parsed = parse(build_file, "test/versioned/path", "BUILD_proto_only.bazel") - self.assertEqual("grpc", parsed.transport) diff --git a/library_generation/test/model/generation_config_unit_tests.py b/library_generation/test/model/generation_config_unit_tests.py deleted file mode 100644 index f6a2f2a2d8..0000000000 --- a/library_generation/test/model/generation_config_unit_tests.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest -from pathlib import Path -from library_generation.model.generation_config import from_yaml, GenerationConfig -from library_generation.model.library_config import LibraryConfig - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") -test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() - -library_1 = LibraryConfig( - api_shortname="a_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], -) -library_2 = LibraryConfig( - api_shortname="another_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], -) -common_protos_library = LibraryConfig( - api_shortname="common-protos", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], -) - - -class GenerationConfigTest(unittest.TestCase): - def test_generation_config_default_value(self): - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[], - ) - self.assertEqual("", config.libraries_bom_version) - - def test_generation_config_with_generator_version_env_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Environment variable GENERATOR_VERSION is not set", - GenerationConfig, - googleapis_commitish="", - libraries=[], - ) - - def test_generation_config_set_generator_version_from_env(self): - os.environ["GENERATOR_VERSION"] = "1.2.3" - config = GenerationConfig( - googleapis_commitish="", - libraries=[], - ) - self.assertEqual("1.2.3", config.gapic_generator_version) - os.environ.pop("GENERATOR_VERSION") - - def test_from_yaml_succeeds(self): - config = from_yaml(f"{test_config_dir}/generation_config.yaml") - self.assertEqual("2.34.0", config.gapic_generator_version) - self.assertEqual(25.2, config.protoc_version) - self.assertEqual( - "1a45bf7393b52407188c82e63101db7dc9c72026", config.googleapis_commitish - ) - self.assertEqual("26.37.0", config.libraries_bom_version) - library = config.libraries[0] - self.assertEqual("cloudasset", library.api_shortname) - self.assertEqual("Cloud Asset Inventory", library.name_pretty) - self.assertEqual( - "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - library.product_documentation, - ) - self.assertEqual( - "provides inventory services based on a time series database.", - library.api_description, - ) - self.assertEqual("asset", library.library_name) - self.assertEqual("@googleapis/analytics-dpe", library.codeowner_team) - self.assertEqual( - "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1", - library.excluded_poms, - ) - self.assertEqual("google-iam-policy", library.excluded_dependencies) - gapics = library.gapic_configs - self.assertEqual(5, len(gapics)) - self.assertEqual("google/cloud/asset/v1", gapics[0].proto_path) - self.assertEqual("google/cloud/asset/v1p1beta1", gapics[1].proto_path) - self.assertEqual("google/cloud/asset/v1p2beta1", gapics[2].proto_path) - self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path) - self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path) - - def test_get_proto_path_to_library_name_success(self): - paths = from_yaml( - f"{test_config_dir}/generation_config.yaml" - ).get_proto_path_to_library_name() - self.assertEqual( - { - "google/cloud/asset/v1": "asset", - "google/cloud/asset/v1p1beta1": "asset", - "google/cloud/asset/v1p2beta1": "asset", - "google/cloud/asset/v1p5beta1": "asset", - "google/cloud/asset/v1p7beta1": "asset", - }, - paths, - ) - - def test_is_monorepo_with_one_library_returns_false(self): - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[library_1], - ) - self.assertFalse(config.is_monorepo()) - - def test_is_monorepo_with_two_libraries_returns_true(self): - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[library_1, library_2], - ) - self.assertTrue(config.is_monorepo()) - - def test_contains_common_protos_with_common_protos_returns_true(self): - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[library_1, library_2, common_protos_library], - ) - self.assertTrue(config.contains_common_protos()) - - def test_contains_common_protos_without_common_protos_returns_false(self): - config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=[library_1, library_2], - ) - self.assertFalse(config.contains_common_protos()) - - def test_validate_with_duplicate_library_name_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "the same library name", - GenerationConfig, - gapic_generator_version="", - googleapis_commitish="", - libraries=[ - LibraryConfig( - api_shortname="secretmanager", - name_pretty="Secret API", - product_documentation="", - api_description="", - gapic_configs=list(), - ), - LibraryConfig( - api_shortname="another-secret", - name_pretty="Another Secret API", - product_documentation="", - api_description="", - gapic_configs=list(), - library_name="secretmanager", - ), - ], - ) - - def test_from_yaml_without_googleapis_commitish_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Repo level parameter, googleapis_commitish", - from_yaml, - f"{test_config_dir}/config_without_googleapis.yaml", - ) - - def test_from_yaml_without_libraries_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Repo level parameter, libraries", - from_yaml, - f"{test_config_dir}/config_without_libraries.yaml", - ) - - def test_from_yaml_without_api_shortname_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Library level parameter, api_shortname", - from_yaml, - f"{test_config_dir}/config_without_api_shortname.yaml", - ) - - def test_from_yaml_without_api_description_raise_exception(self): - self.assertRaisesRegex( - ValueError, - r"Library level parameter, api_description.*'api_shortname': 'apigeeconnect'.*", - from_yaml, - f"{test_config_dir}/config_without_api_description.yaml", - ) - - def test_from_yaml_without_name_pretty_raise_exception(self): - self.assertRaisesRegex( - ValueError, - r"Library level parameter, name_pretty.*'api_shortname': 'apigeeconnect'.*", - from_yaml, - f"{test_config_dir}/config_without_name_pretty.yaml", - ) - - def test_from_yaml_without_product_documentation_raise_exception(self): - self.assertRaisesRegex( - ValueError, - r"Library level parameter, product_documentation.*'api_shortname': 'apigeeconnect'.*", - from_yaml, - f"{test_config_dir}/config_without_product_docs.yaml", - ) - - def test_from_yaml_without_gapics_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Library level parameter, GAPICs.*'api_shortname': 'apigeeconnect'.*", - from_yaml, - f"{test_config_dir}/config_without_gapics_key.yaml", - ) - - def test_from_yaml_without_proto_path_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "GAPIC level parameter, proto_path", - from_yaml, - f"{test_config_dir}/config_without_proto_path.yaml", - ) - - def test_from_yaml_with_zero_library_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Library is None", - from_yaml, - f"{test_config_dir}/config_without_library_value.yaml", - ) - - def test_from_yaml_with_zero_proto_path_raise_exception(self): - self.assertRaisesRegex( - ValueError, - r"GAPICs is None in.*'api_shortname': 'apigeeconnect'.*", - from_yaml, - f"{test_config_dir}/config_without_gapics_value.yaml", - ) diff --git a/library_generation/test/model/library_config_unit_tests.py b/library_generation/test/model/library_config_unit_tests.py deleted file mode 100644 index 5d54737ced..0000000000 --- a/library_generation/test/model/library_config_unit_tests.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.library_config import LibraryConfig - - -class LibraryConfigTest(unittest.TestCase): - def test_get_library_returns_library_name(self): - library = LibraryConfig( - api_shortname="secret", - name_pretty="", - product_documentation="", - api_description="", - gapic_configs=list(), - library_name="secretmanager", - ) - self.assertEqual("secretmanager", library.get_library_name()) - - def test_get_library_returns_api_shortname(self): - library = LibraryConfig( - api_shortname="secret", - name_pretty="", - product_documentation="", - api_description="", - gapic_configs=list(), - ) - self.assertEqual("secret", library.get_library_name()) - - def test_get_sorted_gapic_configs_returns_correct_order(self): - v1beta1 = GapicConfig(proto_path="google/spanner/v1beta1") - v1 = GapicConfig(proto_path="google/spanner/v1") - v1alpha1 = GapicConfig(proto_path="google/spanner/v1alpha") - v2 = GapicConfig(proto_path="google/spanner/v2") - admin_v2 = GapicConfig(proto_path="google/spanner/admin/v2") - non_versioned = GapicConfig(proto_path="google/spanner/type") - library = LibraryConfig( - api_shortname="secret", - name_pretty="", - product_documentation="", - api_description="", - gapic_configs=[v1alpha1, v1, v2, admin_v2, non_versioned, v1beta1], - ) - self.assertEqual( - [ - v2, - v1, - admin_v2, - v1beta1, - v1alpha1, - non_versioned, - ], - library.get_sorted_gapic_configs(), - ) - - def test_init_invalid_distribution_name_raise_value_error(self): - self.assertRaisesRegex( - ValueError, - "fake-distribution-name is not a valid distribution name.", - LibraryConfig, - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - distribution_name="fake-distribution-name", - ) - - def test_get_distribution_name_cloud_api(self): - library = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - ) - self.assertEqual( - "com.google.cloud:google-cloud-baremetalsolution", - library.get_maven_coordinate(), - ) - self.assertEqual("google-cloud-baremetalsolution", library.get_artifact_id()) - - def test_get_distribution_name_non_cloud_api(self): - library = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - cloud_api=False, - group_id="com.example", - ) - self.assertEqual( - "com.example:google-baremetalsolution", library.get_maven_coordinate() - ) - self.assertEqual("google-baremetalsolution", library.get_artifact_id()) - - def test_get_distribution_name_with_distribution_name(self): - library = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - distribution_name="com.example:baremetalsolution", - ) - self.assertEqual( - "com.example:baremetalsolution", library.get_maven_coordinate() - ) - self.assertEqual("baremetalsolution", library.get_artifact_id()) diff --git a/library_generation/test/model/repo_config_unit_tests.py b/library_generation/test/model/repo_config_unit_tests.py deleted file mode 100644 index 12d28fe254..0000000000 --- a/library_generation/test/model/repo_config_unit_tests.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest - -from library_generation.model.repo_config import RepoConfig - -script_dir = os.path.dirname(os.path.realpath(__file__)) -versions_file = os.path.join(script_dir, "..", "resources", "misc", "versions.txt") - - -class RepoConfigTest(unittest.TestCase): - def test_get_library_versions_with_existing_library(self): - repo_config = RepoConfig( - output_folder="test", libraries=dict(), versions_file=versions_file - ) - self.assertEqual( - "2.25.0", - repo_config.get_library_version("gapic-generator-java"), - ) - self.assertEqual( - "2.16.0", - repo_config.get_library_version("api-common"), - ) - self.assertEqual( - "2.33.0", - repo_config.get_library_version("gax"), - ) - self.assertEqual( - "2.34.0", - repo_config.get_library_version("gax-grpc"), - ) - self.assertEqual( - "0.118.0", - repo_config.get_library_version("gax-httpjson"), - ) - - def test_get_library_versions_with_new_library(self): - repo_config = RepoConfig( - output_folder="test", libraries=dict(), versions_file=versions_file - ) - self.assertEqual( - "0.0.0", - repo_config.get_library_version("example-artifact"), - ) diff --git a/library_generation/test/owlbot/__init__.py b/library_generation/test/owlbot/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/owlbot/fix_poms_unit_tests.py b/library_generation/test/owlbot/fix_poms_unit_tests.py deleted file mode 100644 index 496ae49383..0000000000 --- a/library_generation/test/owlbot/fix_poms_unit_tests.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import shutil -import unittest -from library_generation.owlbot.src.fix_poms import main -from library_generation.test.compare_poms import compare_pom_in_subdir - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources", "test-owlbot") - - -class FixPomsTest(unittest.TestCase): - def test_update_poms_group_id_does_not_start_with_google_correctly(self): - ad_manager_resource = os.path.join(resources_dir, "java-admanager") - versions_file = os.path.join(ad_manager_resource, "versions.txt") - os.chdir(ad_manager_resource) - sub_dirs = ["ad-manager", "ad-manager-bom", "proto-ad-manager-v1", "."] - for sub_dir in sub_dirs: - self.__copy__golden(ad_manager_resource, sub_dir) - main(versions_file, "true") - for sub_dir in sub_dirs: - self.assertFalse(compare_pom_in_subdir(ad_manager_resource, sub_dir)) - for sub_dir in sub_dirs: - self.__remove_file_in_subdir(ad_manager_resource, sub_dir) - - @classmethod - def __copy__golden(cls, base_dir: str, subdir: str): - golden = os.path.join(base_dir, subdir, "pom-golden.xml") - pom = os.path.join(base_dir, subdir, "pom.xml") - shutil.copyfile(golden, pom) - - @classmethod - def __remove_file_in_subdir(cls, base_dir: str, subdir: str): - pom = os.path.join(base_dir, subdir, "pom.xml") - os.unlink(pom) diff --git a/library_generation/test/owlbot/java_unit_tests.py b/library_generation/test/owlbot/java_unit_tests.py deleted file mode 100644 index 9d7cb694c5..0000000000 --- a/library_generation/test/owlbot/java_unit_tests.py +++ /dev/null @@ -1,310 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import shutil -import tempfile -import unittest -import xml.etree.ElementTree as elementTree -from unittest import mock - -import yaml -from pathlib import Path -import requests_mock -from synthtool.languages import java -from library_generation.test.owlbot import util - -TEST_OWLBOT = Path(__file__).parent.parent / "resources" / "test-owlbot" -FIXTURES = Path(__file__).parent.parent / "resources" / "test-owlbot" / "fixtures" -TEMPLATES_PATH = Path(__file__).parent.parent.parent / "owlbot" / "templates" - -SAMPLE_METADATA = """ - - com.google.cloud - libraries-bom - - 3.3.0 - 3.3.0 - - 1.0.0 - 1.1.0 - 1.1.1 - 1.2.0 - 2.0.0 - 2.1.0 - 2.2.0 - 2.2.1 - 2.3.0 - 2.4.0 - 2.5.0 - 2.6.0 - 2.7.0 - 2.7.1 - 2.8.0 - 2.9.0 - 3.0.0 - 3.1.0 - 3.1.1 - 3.2.0 - 3.3.0 - - 20191218182827 - - -""" - - -class JavaUnitTests(unittest.TestCase): - - def test_version_from_maven_metadata(self): - self.assertEqual("3.3.0", java.version_from_maven_metadata(SAMPLE_METADATA)) - - def test_latest_maven_version(self): - with requests_mock.Mocker() as m: - m.get( - "https://repo1.maven.org/maven2/com/google/cloud/libraries-bom/maven-metadata.xml", - text=SAMPLE_METADATA, - ) - self.assertEqual( - "3.3.0", - java.latest_maven_version( - group_id="com.google.cloud", artifact_id="libraries-bom" - ), - ) - - @mock.patch.dict(os.environ, {"SYNTHTOOL_TEMPLATES": f"{TEMPLATES_PATH}"}) - def test_working_common_templates(self): - def assert_valid_xml(file): - try: - elementTree.parse(file) - except elementTree.ParseError: - self.fail(f"unable to parse XML: {file}") - - def assert_valid_yaml(file): - with open(file, "r") as stream: - try: - yaml.safe_load(stream) - except yaml.YAMLError: - self.fail(f"unable to parse YAML: {file}") - - with util.copied_fixtures_dir( - FIXTURES / "java_templates" / "standard" - ) as workdir: - # generate the common templates - java.common_templates(template_path=TEMPLATES_PATH) - self.assertTrue(os.path.isfile("renovate.json")) - - # lint xml, yaml files - # use os.walk because glob ignores hidden directories - for dirpath, _, filenames in os.walk(workdir): - for file in filenames: - (_, ext) = os.path.splitext(file) - if ext == ".xml": - assert_valid_xml(os.path.join(dirpath, file)) - elif ext == ".yaml" or ext == ".yml": - assert_valid_yaml(os.path.join(dirpath, file)) - - def test_remove_method(self): - with tempfile.TemporaryDirectory() as tempdir: - cwd = os.getcwd() - os.chdir(TEST_OWLBOT) - shutil.copyfile("testdata/SampleClass.java", tempdir + "/SampleClass.java") - - java.remove_method( - tempdir + "/SampleClass.java", "public static void foo()" - ) - java.remove_method(tempdir + "/SampleClass.java", "public void asdf()") - self.assert_matches_golden( - "testdata/SampleClassGolden.java", tempdir + "/SampleClass.java" - ) - os.chdir(cwd) - - def test_copy_and_rename_method(self): - with tempfile.TemporaryDirectory() as tempdir: - cwd = os.getcwd() - os.chdir(TEST_OWLBOT) - shutil.copyfile("testdata/SampleClass.java", tempdir + "/SampleClass.java") - - java.copy_and_rename_method( - tempdir + "/SampleClass.java", - "public static void foo()", - "foo", - "foobar", - ) - java.copy_and_rename_method( - tempdir + "/SampleClass.java", "public void asdf()", "asdf", "xyz" - ) - self.assert_matches_golden( - "testdata/SampleCopyMethodGolden.java", - tempdir + "/SampleClass.java", - ) - os.chdir(cwd) - - def test_deprecate_method(self): - with tempfile.TemporaryDirectory() as tempdir: - cwd = os.getcwd() - os.chdir(TEST_OWLBOT) - shutil.copyfile( - "testdata/SampleDeprecateClass.java", - tempdir + "/SampleDeprecateClass.java", - ) - deprecation_warning = """This method will be removed in the next major version.\nUse {{@link #{new_method}()}} instead""" - additional_comment = ( - """{new_method} has the same functionality as foobar.""" - ) - java.deprecate_method( - tempdir + "/SampleDeprecateClass.java", - "public void foo(String bar)", - deprecation_warning.format(new_method="sample"), - ) - - # adding a comment when a javadoc and annotation already exists - java.deprecate_method( - tempdir + "/SampleDeprecateClass.java", - "public void bar(String bar)", - deprecation_warning.format(new_method="sample"), - ) - java.deprecate_method( - tempdir + "/SampleDeprecateClass.java", - "public void cat(String bar)", - additional_comment.format(new_method="sample"), - ) - - self.assert_matches_golden( - "testdata/SampleDeprecateMethodGolden.java", - tempdir + "/SampleDeprecateClass.java", - ) - os.chdir(cwd) - - def test_fix_proto_license(self): - with tempfile.TemporaryDirectory() as tempdir: - cwd = os.getcwd() - os.chdir(TEST_OWLBOT) - temppath = Path(tempdir).resolve() - os.mkdir(temppath / "src") - shutil.copyfile( - "testdata/src/foo/FooProto.java", temppath / "src/FooProto.java" - ) - - java.fix_proto_headers(temppath) - self.assert_matches_golden( - "testdata/FooProtoGolden.java", temppath / "src/FooProto.java" - ) - os.chdir(cwd) - - def test_fix_proto_license_idempotent(self): - with tempfile.TemporaryDirectory() as tempdir: - cwd = os.getcwd() - os.chdir(TEST_OWLBOT) - temppath = Path(tempdir).resolve() - os.mkdir(temppath / "src") - shutil.copyfile( - "testdata/src/foo/FooProto.java", temppath / "src/FooProto.java" - ) - - # run the header fix twice - java.fix_proto_headers(temppath) - java.fix_proto_headers(temppath) - self.assert_matches_golden( - "testdata/FooProtoGolden.java", temppath / "src/FooProto.java" - ) - os.chdir(cwd) - - def test_fix_grpc_license(self): - with tempfile.TemporaryDirectory() as tempdir: - cwd = os.getcwd() - os.chdir(TEST_OWLBOT) - temppath = Path(tempdir).resolve() - os.mkdir(temppath / "src") - shutil.copyfile( - "testdata/src/foo/FooGrpc.java", temppath / "src/FooGrpc.java" - ) - - java.fix_grpc_headers(temppath) - self.assert_matches_golden( - "testdata/FooGrpcGolden.java", temppath / "src/FooGrpc.java" - ) - os.chdir(cwd) - - def test_fix_grpc_license_idempotent(self): - with tempfile.TemporaryDirectory() as tempdir: - cwd = os.getcwd() - os.chdir(TEST_OWLBOT) - temppath = Path(tempdir).resolve() - os.mkdir(temppath / "src") - shutil.copyfile( - "testdata/src/foo/FooGrpc.java", temppath / "src/FooGrpc.java" - ) - - # run the header fix twice - java.fix_grpc_headers(temppath) - java.fix_grpc_headers(temppath) - self.assert_matches_golden( - "testdata/FooGrpcGolden.java", temppath / "src/FooGrpc.java" - ) - os.chdir(cwd) - - @mock.patch.dict(os.environ, {"SYNTHTOOL_TEMPLATES": f"{TEMPLATES_PATH}"}) - def test_release_please_handle_releases(self): - with util.copied_fixtures_dir( - FIXTURES / "java_templates" / "release-please-update" - ): - # generate the common templates - java.common_templates(template_path=TEMPLATES_PATH) - - self.assertTrue(os.path.isfile(".github/release-please.yml")) - with open(".github/release-please.yml") as fp: - self.assertEqual( - fp.read(), - """branches: -- branch: 1.127.12-sp - bumpMinorPreMajor: true - handleGHRelease: true - releaseType: java-lts -bumpMinorPreMajor: true -handleGHRelease: true -releaseType: java-yoshi -""", - ) - - @mock.patch.dict( - os.environ, - { - "SYNTHTOOL_TEMPLATES": f"{TEMPLATES_PATH}", - "SYNTHTOOL_LIBRARY_VERSION": "1.2.3", - }, - ) - def test_render_readme_success(self): - golden_path = os.path.abspath(f"{TEST_OWLBOT}/testdata/README-golden.md") - with util.copied_fixtures_dir(FIXTURES / "java_templates" / "render-readme"): - # This method needs read .repo-metadata.json to render templates. - # The file is located in FIXTURES/java_templates/render-readme. - java.common_templates( - template_path=TEMPLATES_PATH, - ) - self.assertTrue(os.path.isfile("README.md")) - self.assert_matches_golden(golden_path, "README.md") - - def assert_matches_golden(self, expected, actual): - matching_lines = 0 - with open(actual, "rt") as fp: - with open(expected, "rt") as golden: - while True: - matching_lines += 1 - log_line = fp.readline() - expected = golden.readline() - self.assertEqual(repr(log_line), repr(expected)) - if not log_line: - break - assert matching_lines > 0 diff --git a/library_generation/test/owlbot/util.py b/library_generation/test/owlbot/util.py deleted file mode 100644 index da54846768..0000000000 --- a/library_generation/test/owlbot/util.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import contextlib -import subprocess -import os -import pathlib -import shutil -import sys -import tempfile -import typing - - -def make_working_repo(working_dir: str, default_branch: str = "main"): - """Create a local repo that resembles a real repo. - - Specifically, it has a history of synth.py changes that actually change the - generated output. - """ - subprocess.check_call(["git", "init"], cwd=working_dir) - subprocess.check_call(["git", "checkout", "-b", default_branch], cwd=working_dir) - subprocess.check_call( - [ - "git", - "remote", - "add", - "origin", - "https://github.com/googleapis/nodejs-vision.git", - ] - ) - working_path = pathlib.Path(working_dir) - # The simplest possible synth.py. It generates one file with one line of text. - template = """import time -import json -import uuid -import subprocess - -# comment - -with open("generated.txt", "wt") as f: - f.write("a\\n") -metadata = { "updateTime": str(uuid.uuid4()), - "sources": [ - { - "git": { - "name": ".", - "remote": "https://github.com/googleapis/synthtool.git", - "sha": subprocess.run(["git", "log", "-1", "--pretty=%H"], universal_newlines=True, stdout=subprocess.PIPE).stdout.strip(), - } - }] -} -with open("synth.metadata", "wt") as f: - json.dump(metadata, f) -""" - # Write version a. - synth_py_path = working_path / "synth.py" - synth_py_path.write_text(template) - subprocess.check_call([sys.executable, str(synth_py_path)], cwd=working_dir) - subprocess.check_call(["git", "add", "-A"], cwd=working_dir) - subprocess.check_call(["git", "commit", "-m", "a"], cwd=working_dir) - - # Write version b. - text = template.replace('"a\\n"', '"b\\n"') - synth_py_path.write_text(text) - subprocess.check_call(["git", "commit", "-am", "b"], cwd=working_dir) - - # Write a version that has no effect on output. - text = text.replace("# comment", "# a different comment") - synth_py_path.write_text(text) - subprocess.check_call(["git", "commit", "-am", "comment"], cwd=working_dir) - - # Write version c. - text = text.replace('"b\\n"', '"c\\n"') - synth_py_path.write_text(text) - subprocess.check_call( - ["git", "commit", "-am", "c subject\n\nbody line 1\nbody line 2"], - cwd=working_dir, - ) - - return text - - -@contextlib.contextmanager -def chdir(path: typing.Union[pathlib.Path, str]): - """Context Manager to change the current working directory and restore the - previous working directory after completing the context. - - Args: - path (pathlib.Path, str) - The new current working directory. - Yields: - pathlib.Path - The new current working directory. - """ - old_cwd = os.getcwd() - os.chdir(str(path)) - try: - yield pathlib.Path(path) - finally: - os.chdir(old_cwd) - - -@contextlib.contextmanager -def copied_fixtures_dir(source: pathlib.Path): - """Context Manager to copy from a fixtures directory into a new temporary directory - and change the current working directory to that copy. Restores the original - current working directory after completing the context. - - Args: - source (pathlib.Path) - The directory to copy. - Yields: - pathlib.Path - The temporary directory with the copied contents. - """ - with tempfile.TemporaryDirectory() as tempdir: - workdir = shutil.copytree(source, pathlib.Path(tempdir) / "workspace") - with chdir(workdir): - yield workdir diff --git a/library_generation/test/resources/gapic_options/QueryServiceGrpc_copy.java b/library_generation/test/resources/gapic_options/QueryServiceGrpc_copy.java deleted file mode 100644 index cf2bf2b0ee..0000000000 --- a/library_generation/test/resources/gapic_options/QueryServiceGrpc_copy.java +++ /dev/null @@ -1,9 +0,0 @@ -/** - * This file is only used in testing `remove_grpc_version` in utilities.sh. - */ -@javax.annotation.processing.Generated( - value = "by gRPC proto compiler 1.55.1", - comments = "Source: google/monitoring/v3/query_service.proto") -public final class QueryServiceGrpc_copy { - -} diff --git a/library_generation/test/resources/gapic_options/example.yaml b/library_generation/test/resources/gapic_options/example.yaml deleted file mode 100644 index 7bd824910e..0000000000 --- a/library_generation/test/resources/gapic_options/example.yaml +++ /dev/null @@ -1 +0,0 @@ -# this file is only used in testing `get_gapic_opts` in utilities.sh diff --git a/library_generation/test/resources/gapic_options/example_gapic.legacy.yaml b/library_generation/test/resources/gapic_options/example_gapic.legacy.yaml deleted file mode 100644 index 7da8426f34..0000000000 --- a/library_generation/test/resources/gapic_options/example_gapic.legacy.yaml +++ /dev/null @@ -1 +0,0 @@ -# this file is only used in testing `get_gapic_opts` in utilities.sh \ No newline at end of file diff --git a/library_generation/test/resources/gapic_options/example_gapic.yaml b/library_generation/test/resources/gapic_options/example_gapic.yaml deleted file mode 100644 index 7bd824910e..0000000000 --- a/library_generation/test/resources/gapic_options/example_gapic.yaml +++ /dev/null @@ -1 +0,0 @@ -# this file is only used in testing `get_gapic_opts` in utilities.sh diff --git a/library_generation/test/resources/gapic_options/example_gapic_legacy.yaml b/library_generation/test/resources/gapic_options/example_gapic_legacy.yaml deleted file mode 100644 index 7da8426f34..0000000000 --- a/library_generation/test/resources/gapic_options/example_gapic_legacy.yaml +++ /dev/null @@ -1 +0,0 @@ -# this file is only used in testing `get_gapic_opts` in utilities.sh \ No newline at end of file diff --git a/library_generation/test/resources/gapic_options/example_grpc_service_config.json b/library_generation/test/resources/gapic_options/example_grpc_service_config.json deleted file mode 100644 index 947c390835..0000000000 --- a/library_generation/test/resources/gapic_options/example_grpc_service_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "comment": "this file is only used in testing `get_gapic_opts` in utilities.sh" -} diff --git a/library_generation/test/resources/goldens/.OwlBot-hermetic-golden.yaml b/library_generation/test/resources/goldens/.OwlBot-hermetic-golden.yaml deleted file mode 100644 index 225b4620bf..0000000000 --- a/library_generation/test/resources/goldens/.OwlBot-hermetic-golden.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -deep-remove-regex: -- "/java-bare-metal-solution/grpc-google-.*/src" -- "/java-bare-metal-solution/proto-google-.*/src" -- "/java-bare-metal-solution/google-.*/src" -- "/java-bare-metal-solution/samples/snippets/generated" - -deep-preserve-regex: -- "/java-bare-metal-solution/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" - -deep-copy-regex: -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/proto-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/proto-google-cloud-bare-metal-solution-$1/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/grpc-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/grpc-google-cloud-bare-metal-solution-$1/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/gapic-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/google-cloud-bare-metal-solution/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/samples/snippets/generated" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/samples/snippets/generated" - -api-name: baremetalsolution \ No newline at end of file diff --git a/library_generation/test/resources/goldens/.repo-metadata-custom-transport-golden.json b/library_generation/test/resources/goldens/.repo-metadata-custom-transport-golden.json deleted file mode 100644 index 2cccd4b889..0000000000 --- a/library_generation/test/resources/goldens/.repo-metadata-custom-transport-golden.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "api_shortname": "secretmanager", - "name_pretty": "Secret Management", - "product_documentation": "https://cloud.google.com/solutions/secrets-management/", - "api_description": "allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-secretmanager/latest/overview", - "release_level": "preview", - "transport": "http", - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-secretmanager", - "distribution_name": "com.google.cloud:google-cloud-secretmanager", - "api_id": "secretmanager.googleapis.com", - "library_type": "GAPIC_AUTO", - "requires_billing": true -} \ No newline at end of file diff --git a/library_generation/test/resources/goldens/.repo-metadata-monorepo-golden.json b/library_generation/test/resources/goldens/.repo-metadata-monorepo-golden.json deleted file mode 100644 index 0b3bbc2b55..0000000000 --- a/library_generation/test/resources/goldens/.repo-metadata-monorepo-golden.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "api_id": "baremetalsolution.googleapis.com", - "library_type": "GAPIC_AUTO", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", - "recommended_package": "com.google.example", - "min_java_version": 8 -} \ No newline at end of file diff --git a/library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json b/library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json deleted file mode 100644 index b39c297a85..0000000000 --- a/library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/java-bare-metal-solution", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "api_id": "baremetalsolution.googleapis.com", - "library_type": "GAPIC_COMBO", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", - "extra_versioned_modules": "test-module", - "recommended_package": "com.google.example", - "min_java_version": 8 -} \ No newline at end of file diff --git a/library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json b/library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json deleted file mode 100644 index 7730cd6987..0000000000 --- a/library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/sdk-platform-java", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "library_type": "OTHER", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", - "recommended_package": "com.google.example", - "min_java_version": 8 -} \ No newline at end of file diff --git a/library_generation/test/resources/goldens/owlbot-golden.py b/library_generation/test/resources/goldens/owlbot-golden.py deleted file mode 100644 index 2ba11e6bba..0000000000 --- a/library_generation/test/resources/goldens/owlbot-golden.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import synthtool as s -from synthtool.languages import java - - -for library in s.get_staging_dirs(): - # put any special-case replacements here - s.move(library) - -s.remove_staging_dirs() -java.common_templates(monorepo=True, excludes=[ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "SECURITY.md", - "java.header", - "license-checks.xml", - "renovate.json", - ".gitignore" -]) \ No newline at end of file diff --git a/library_generation/test/resources/goldens/pr_description-golden.txt b/library_generation/test/resources/goldens/pr_description-golden.txt deleted file mode 100644 index 1a0f874936..0000000000 --- a/library_generation/test/resources/goldens/pr_description-golden.txt +++ /dev/null @@ -1,17 +0,0 @@ -This pull request is generated with proto changes between [googleapis/googleapis@3b6f144](https://github.com/googleapis/googleapis/commit/3b6f144d47b0a1d2115ab2445ec06e80cc324a44) (exclusive) and [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) (inclusive). - -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -chore: update the libraries_bom version to 2.3.4 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: Make Layout Parser generally available in V1 - -PiperOrigin-RevId: 638924855 - -Source Link: [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/library_generation/test/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt b/library_generation/test/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt deleted file mode 100644 index 81f7c0c4d2..0000000000 --- a/library_generation/test/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt +++ /dev/null @@ -1,10 +0,0 @@ -This pull request is generated with proto changes between [googleapis/googleapis@3b6f144](https://github.com/googleapis/googleapis/commit/3b6f144d47b0a1d2115ab2445ec06e80cc324a44) (exclusive) and [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) (inclusive). - -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -chore: update the libraries_bom version to 2.3.4 -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/library_generation/test/resources/goldens/repo_level_only_pr_description-golden.txt b/library_generation/test/resources/goldens/repo_level_only_pr_description-golden.txt deleted file mode 100644 index f89901f8c1..0000000000 --- a/library_generation/test/resources/goldens/repo_level_only_pr_description-golden.txt +++ /dev/null @@ -1,5 +0,0 @@ -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml deleted file mode 100644 index 9a27dd381d..0000000000 --- a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml +++ /dev/null @@ -1,45 +0,0 @@ -gapic_generator_version: 2.38.1 -protoc_version: 25.2 -googleapis_commitish: a17d4caf184b050d50cacf2b0d579ce72c31ce74 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." - release_level: "stable" - library_name: "apigee-connect" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 - - - api_shortname: alloydb - name_pretty: AlloyDB - product_documentation: https://cloud.google.com/alloydb/ - api_description: AlloyDB is a fully managed, PostgreSQL-compatible database service - with industry-leading performance, availability, and scale. - rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest - GAPICs: - - proto_path: google/cloud/alloydb/v1 - - proto_path: google/cloud/alloydb/v1alpha - - proto_path: google/cloud/alloydb/v1beta - - - api_shortname: alloydb - name_pretty: AlloyDB connectors - product_documentation: https://cloud.google.com/alloydb/docs - api_description: AlloyDB is a fully-managed, PostgreSQL-compatible database for - demanding transactional workloads. It provides enterprise-grade performance and - availability while maintaining 100% compatibility with open-source PostgreSQL. - library_name: alloydb-connectors - rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest - GAPICs: - - proto_path: google/cloud/alloydb/connectors/v1 - - proto_path: google/cloud/alloydb/connectors/v1alpha - - proto_path: google/cloud/alloydb/connectors/v1beta - - - api_shortname: cloudcontrolspartner - name_pretty: Cloud Controls Partner API - product_documentation: https://cloud.google.com/sovereign-controls-by-partners/docs/sovereign-partners - api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. - GAPICs: - - proto_path: google/cloud/cloudcontrolspartner/v1 - - proto_path: google/cloud/cloudcontrolspartner/v1beta diff --git a/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml deleted file mode 100644 index adc1d170d6..0000000000 --- a/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml +++ /dev/null @@ -1,45 +0,0 @@ -gapic_generator_version: 2.38.1 -protoc_version: 25.2 -googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." - release_level: "stable" - library_name: "apigee-connect" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 - - - api_shortname: alloydb - name_pretty: AlloyDB - product_documentation: https://cloud.google.com/alloydb/ - api_description: AlloyDB is a fully managed, PostgreSQL-compatible database service - with industry-leading performance, availability, and scale. - rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest - GAPICs: - - proto_path: google/cloud/alloydb/v1 - - proto_path: google/cloud/alloydb/v1alpha - - proto_path: google/cloud/alloydb/v1beta - - - api_shortname: alloydb - name_pretty: AlloyDB connectors - product_documentation: https://cloud.google.com/alloydb/docs - api_description: AlloyDB is a fully-managed, PostgreSQL-compatible database for - demanding transactional workloads. It provides enterprise-grade performance and - availability while maintaining 100% compatibility with open-source PostgreSQL. - library_name: alloydb-connectors - rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest - GAPICs: - - proto_path: google/cloud/alloydb/connectors/v1 - - proto_path: google/cloud/alloydb/connectors/v1alpha - - proto_path: google/cloud/alloydb/connectors/v1beta - - - api_shortname: cloudcontrolspartner - name_pretty: Cloud Controls Partner API - product_documentation: https://cloud.google.com/sovereign-controls-by-partners/docs/sovereign-partners - api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. - GAPICs: - - proto_path: google/cloud/cloudcontrolspartner/v1 - - proto_path: google/cloud/cloudcontrolspartner/v1beta diff --git a/library_generation/test/resources/integration/google-cloud-java/pr-description-golden.txt b/library_generation/test/resources/integration/google-cloud-java/pr-description-golden.txt deleted file mode 100644 index cb855dcd52..0000000000 --- a/library_generation/test/resources/integration/google-cloud-java/pr-description-golden.txt +++ /dev/null @@ -1,54 +0,0 @@ -This pull request is generated with proto changes between [googleapis/googleapis@a17d4ca](https://github.com/googleapis/googleapis/commit/a17d4caf184b050d50cacf2b0d579ce72c31ce74) (exclusive) and [googleapis/googleapis@4ce0ff6](https://github.com/googleapis/googleapis/commit/4ce0ff67a3d4509be641cbe47a35844ddc1268fc) (inclusive). - -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -docs: [cloudcontrolspartner] update documentation URL - -PiperOrigin-RevId: 612723053 - -Source Link: [googleapis/googleapis@7659dd2](https://github.com/googleapis/googleapis/commit/7659dd2244563fd902b681bdcadbe5385b8d3462) -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: [cloudcontrolspartner] added CloudControlsPartner API - -PiperOrigin-RevId: 612632640 - -Source Link: [googleapis/googleapis@05d889e](https://github.com/googleapis/googleapis/commit/05d889e7dfe087fc2ddc9de9579f01d4e1c2f35e) -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: [alloydb] support for obtaining the public IP address of an Instance -feat: [alloydb] support for getting PSC DNS name from the GetConnectionInfo API -feat: [alloydb] add PSC cluster and instance configuration settings to enable/disable PSC and obtain the PSC endpoint name -feat: [alloydb] add new API to list the databases in a project and location -docs: [alloydb] clarified read pool config is for read pool type instances - -PiperOrigin-RevId: 610475013 - -Source Link: [googleapis/googleapis@aa16fda](https://github.com/googleapis/googleapis/commit/aa16fdad909bc33e2d4ff04cfde56a46d0e52b13) -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: [alloydb] support for obtaining the public IP address of an Instance -feat: [alloydb] support for getting PSC DNS name from the GetConnectionInfo API - -PiperOrigin-RevId: 610415824 - -Source Link: [googleapis/googleapis@0733fdb](https://github.com/googleapis/googleapis/commit/0733fdb5f745192f9f3c95f8d08039286567cbcc) -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -docs: [cloudcontrolspartner] Updated comment for method `ListCustomers` in service `CloudControlsPartnerCore` -docs: [cloudcontrolspartner] Updated comment for field `location` in message `.google.cloud.cloudcontrolspartner.v1beta.Workload` -docs: [cloudcontrolspartner] Updated a comment for field `partner_project_id` in message `.google.cloud.cloudcontrolspartner.v1beta.Partner` -docs: [cloudcontrolspartner] Updated documentation URL - -PiperOrigin-RevId: 609026905 - -Source Link: [googleapis/googleapis@9e35c62](https://github.com/googleapis/googleapis/commit/9e35c620157d7b11cb5b2e5d0249c5caaee824f3) -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: [cloudcontrolspartner] added CloudControlsPartner API - -PiperOrigin-RevId: 606720708 - -Source Link: [googleapis/googleapis@36dedd0](https://github.com/googleapis/googleapis/commit/36dedd0d9020c19d1c8259003c2fe9656ada7471) -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml deleted file mode 100644 index c6912c8125..0000000000 --- a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -gapic_generator_version: 2.37.0 -protoc_version: 25.2 -googleapis_commitish: 9868a57470a969ffa1d21194a5c05d7a6e4e98cc -libraries: -- api_shortname: bigtable - name_pretty: Cloud Bigtable - api_description: "Java idiomatic client for Cloud Bigtable." - product_documentation: "https://cloud.google.com/bigtable" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-bigtable/latest/history" - issue_tracker: "https://issuetracker.google.com/savedsearches/559777" - release_level: "stable" - language: "java" - repo: "googleapis/java-bigtable" - repo_short: "java-bigtable" - distribution_name: "com.google.cloud:google-cloud-bigtable" - excluded_poms: "google-cloud-bigtable-bom" - extra_versioned_modules: "google-cloud-bigtable-emulator,google-cloud-bigtable-emulator-core" - codeowner_team: "@googleapis/api-bigtable @googleapis/api-bigtable-partners" - library_type: GAPIC_COMBO - GAPICs: - - proto_path: google/bigtable/admin/v2 - - proto_path: google/bigtable/v2 diff --git a/library_generation/test/resources/integration/java-bigtable/pr-description-golden.txt b/library_generation/test/resources/integration/java-bigtable/pr-description-golden.txt deleted file mode 100644 index 2dcdbc7756..0000000000 --- a/library_generation/test/resources/integration/java-bigtable/pr-description-golden.txt +++ /dev/null @@ -1,18 +0,0 @@ -This pull request is generated with proto changes between googleapis commit 679060c64136e85b52838f53cfe612ce51e60d1d (exclusive) and fc3043ebe12fb6bc1729c175e1526c859ce751d8 (inclusive). -Qualified commits are: -[googleapis/googleapis@fbcfef0](https://github.com/googleapis/googleapis/commit/fbcfef09510b842774530989889ed1584a8b5acb) -[googleapis/googleapis@63d2a60](https://github.com/googleapis/googleapis/commit/63d2a60056ad5b156c05c7fb13138fc886c3b739) -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix: extend timeouts for deleting snapshots, backups and tables - -PiperOrigin-RevId: 605388988 - -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -chore: update retry settings for backup rpcs - -PiperOrigin-RevId: 605367937 - -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/library_generation/test/resources/integration/test_generator_coordinates b/library_generation/test/resources/integration/test_generator_coordinates deleted file mode 100644 index 00af5f647f..0000000000 --- a/library_generation/test/resources/integration/test_generator_coordinates +++ /dev/null @@ -1 +0,0 @@ -com.google.api:gapic-generator-java:2.38.1 \ No newline at end of file diff --git a/library_generation/test/resources/misc/BUILD_comment_common_resources.bazel b/library_generation/test/resources/misc/BUILD_comment_common_resources.bazel deleted file mode 100644 index 126ffdb7ca..0000000000 --- a/library_generation/test/resources/misc/BUILD_comment_common_resources.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - #"//google/cloud:common_resources_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/misc/BUILD_comment_iam_policy.bazel b/library_generation/test/resources/misc/BUILD_comment_iam_policy.bazel deleted file mode 100644 index a9a2c1ca75..0000000000 --- a/library_generation/test/resources/misc/BUILD_comment_iam_policy.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - # "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/misc/BUILD_comment_locations.bazel b/library_generation/test/resources/misc/BUILD_comment_locations.bazel deleted file mode 100644 index 8b96e3ab81..0000000000 --- a/library_generation/test/resources/misc/BUILD_comment_locations.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - # "//google/cloud/location:location_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/misc/BUILD_common_resources.bazel b/library_generation/test/resources/misc/BUILD_common_resources.bazel deleted file mode 100644 index 9b749e6ad5..0000000000 --- a/library_generation/test/resources/misc/BUILD_common_resources.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/cloud:common_resources_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/misc/BUILD_gapic_yaml.bazel b/library_generation/test/resources/misc/BUILD_gapic_yaml.bazel deleted file mode 100644 index b55f4550d8..0000000000 --- a/library_generation/test/resources/misc/BUILD_gapic_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - gapic_yaml = "test_gapic_yaml.yaml", -) diff --git a/library_generation/test/resources/misc/BUILD_grpc.bazel b/library_generation/test/resources/misc/BUILD_grpc.bazel deleted file mode 100644 index 3e59a9bd35..0000000000 --- a/library_generation/test/resources/misc/BUILD_grpc.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_transport_from_BUILD` in utilities.sh - -java_gapic_library( - transport = "grpc", -) diff --git a/library_generation/test/resources/misc/BUILD_grpc_rest.bazel b/library_generation/test/resources/misc/BUILD_grpc_rest.bazel deleted file mode 100644 index 8a98e7e646..0000000000 --- a/library_generation/test/resources/misc/BUILD_grpc_rest.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_transport_from_BUILD` in utilities.sh - -java_gapic_library( - transport = "grpc+rest", -) diff --git a/library_generation/test/resources/misc/BUILD_iam_locations.bazel b/library_generation/test/resources/misc/BUILD_iam_locations.bazel deleted file mode 100644 index d0c971da7c..0000000000 --- a/library_generation/test/resources/misc/BUILD_iam_locations.bazel +++ /dev/null @@ -1,6 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/cloud/location:location_proto", - "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/misc/BUILD_iam_policy.bazel b/library_generation/test/resources/misc/BUILD_iam_policy.bazel deleted file mode 100644 index af5d4a32f8..0000000000 --- a/library_generation/test/resources/misc/BUILD_iam_policy.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/misc/BUILD_include_samples_empty.bazel b/library_generation/test/resources/misc/BUILD_include_samples_empty.bazel deleted file mode 100644 index b9d4cd56e0..0000000000 --- a/library_generation/test/resources/misc/BUILD_include_samples_empty.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh - -java_gapic_assembly_gradle_pkg( - -) diff --git a/library_generation/test/resources/misc/BUILD_include_samples_false.bazel b/library_generation/test/resources/misc/BUILD_include_samples_false.bazel deleted file mode 100644 index 8e95c3d2a6..0000000000 --- a/library_generation/test/resources/misc/BUILD_include_samples_false.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh - -java_gapic_assembly_gradle_pkg( - include_samples = False, -) diff --git a/library_generation/test/resources/misc/BUILD_include_samples_true.bazel b/library_generation/test/resources/misc/BUILD_include_samples_true.bazel deleted file mode 100644 index bac72b678f..0000000000 --- a/library_generation/test/resources/misc/BUILD_include_samples_true.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh - -java_gapic_assembly_gradle_pkg( - include_samples = True, -) diff --git a/library_generation/test/resources/misc/BUILD_locations.bazel b/library_generation/test/resources/misc/BUILD_locations.bazel deleted file mode 100644 index 29ee14fdba..0000000000 --- a/library_generation/test/resources/misc/BUILD_locations.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/cloud/location:location_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/misc/BUILD_no_additional_protos.bazel b/library_generation/test/resources/misc/BUILD_no_additional_protos.bazel deleted file mode 100644 index a22257cad4..0000000000 --- a/library_generation/test/resources/misc/BUILD_no_additional_protos.bazel +++ /dev/null @@ -1,4 +0,0 @@ -proto_library_with_info( - deps = [ - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/misc/BUILD_no_gapic_yaml.bazel b/library_generation/test/resources/misc/BUILD_no_gapic_yaml.bazel deleted file mode 100644 index 1e9462aa30..0000000000 --- a/library_generation/test/resources/misc/BUILD_no_gapic_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - gapic_yaml = None -) diff --git a/library_generation/test/resources/misc/BUILD_no_service_config.bazel b/library_generation/test/resources/misc/BUILD_no_service_config.bazel deleted file mode 100644 index dbde6de05c..0000000000 --- a/library_generation/test/resources/misc/BUILD_no_service_config.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - grpc_service_config = None -) diff --git a/library_generation/test/resources/misc/BUILD_no_service_yaml.bazel b/library_generation/test/resources/misc/BUILD_no_service_yaml.bazel deleted file mode 100644 index 05bae16d5d..0000000000 --- a/library_generation/test/resources/misc/BUILD_no_service_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - service_yaml = None -) diff --git a/library_generation/test/resources/misc/BUILD_proto_only.bazel b/library_generation/test/resources/misc/BUILD_proto_only.bazel deleted file mode 100644 index 26bcea6126..0000000000 --- a/library_generation/test/resources/misc/BUILD_proto_only.bazel +++ /dev/null @@ -1,16 +0,0 @@ -java_gapic_assembly_gradle_pkg( - name = "google-api-java", - transport = "grpc+rest", - deps = [ - "annotations_proto", - "auth_proto", - "backend_proto", - "billing_proto", - "client_proto", - "config_change_proto", - "consumer_proto", - "context_proto", - "control_proto", - "distribution_proto", - ], -) \ No newline at end of file diff --git a/library_generation/test/resources/misc/BUILD_rest.bazel b/library_generation/test/resources/misc/BUILD_rest.bazel deleted file mode 100644 index 9dff694297..0000000000 --- a/library_generation/test/resources/misc/BUILD_rest.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_transport_from_BUILD` in utilities.sh - -java_gapic_library( - transport = "rest", -) diff --git a/library_generation/test/resources/misc/BUILD_rest_numeric_enums_empty.bazel b/library_generation/test/resources/misc/BUILD_rest_numeric_enums_empty.bazel deleted file mode 100644 index 992b91e52c..0000000000 --- a/library_generation/test/resources/misc/BUILD_rest_numeric_enums_empty.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh - -java_gapic_library( - -) diff --git a/library_generation/test/resources/misc/BUILD_rest_numeric_enums_false.bazel b/library_generation/test/resources/misc/BUILD_rest_numeric_enums_false.bazel deleted file mode 100644 index a446c6b2a7..0000000000 --- a/library_generation/test/resources/misc/BUILD_rest_numeric_enums_false.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh - -java_gapic_library( - rest_numeric_enums = False -) diff --git a/library_generation/test/resources/misc/BUILD_rest_numeric_enums_true.bazel b/library_generation/test/resources/misc/BUILD_rest_numeric_enums_true.bazel deleted file mode 100644 index c4d7fefeb4..0000000000 --- a/library_generation/test/resources/misc/BUILD_rest_numeric_enums_true.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh - -java_gapic_library( - rest_numeric_enums = True -) diff --git a/library_generation/test/resources/misc/BUILD_service_config.bazel b/library_generation/test/resources/misc/BUILD_service_config.bazel deleted file mode 100644 index 097d1bb6bd..0000000000 --- a/library_generation/test/resources/misc/BUILD_service_config.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - grpc_service_config = "test_service_config.json" -) diff --git a/library_generation/test/resources/misc/BUILD_service_config_relative_target.bazel b/library_generation/test/resources/misc/BUILD_service_config_relative_target.bazel deleted file mode 100644 index ccd59af2fb..0000000000 --- a/library_generation/test/resources/misc/BUILD_service_config_relative_target.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - grpc_service_config = ":compute_grpc_service_config.json" -) diff --git a/library_generation/test/resources/misc/BUILD_service_yaml.bazel b/library_generation/test/resources/misc/BUILD_service_yaml.bazel deleted file mode 100644 index f7e4c91f4e..0000000000 --- a/library_generation/test/resources/misc/BUILD_service_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - service_yaml = "test_service_yaml.yaml" -) diff --git a/library_generation/test/resources/misc/BUILD_service_yaml_absolute_target.bazel b/library_generation/test/resources/misc/BUILD_service_yaml_absolute_target.bazel deleted file mode 100644 index ded899dff7..0000000000 --- a/library_generation/test/resources/misc/BUILD_service_yaml_absolute_target.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - service_yaml = "//google/cloud/videointelligence:videointelligence_v1p3beta1.yaml", -) \ No newline at end of file diff --git a/library_generation/test/resources/misc/TESTWORKSPACE b/library_generation/test/resources/misc/TESTWORKSPACE deleted file mode 100644 index 60b3036d9d..0000000000 --- a/library_generation/test/resources/misc/TESTWORKSPACE +++ /dev/null @@ -1,133 +0,0 @@ -# test workspace file obtained from sdk-platform-java - -workspace(name = "gapic_generator_java") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -# gax-java and its transitive dependencies must be imported before -# gapic-generator-java dependencies to match the order in googleapis repository, -# which in its turn, prioritizes actual generated clients runtime dependencies -# over the generator dependencies. -local_repository( - name = "com_google_api_gax_java", - path = "gax-java", -) - -load("@com_google_api_gax_java//:repository_rules.bzl", "com_google_api_gax_java_properties") - -com_google_api_gax_java_properties( - name = "com_google_api_gax_java_properties", - file = "@com_google_api_gax_java//:dependencies.properties", -) - -load("@com_google_api_gax_java//:repositories.bzl", "com_google_api_gax_java_repositories") - -com_google_api_gax_java_repositories() - -_googleapis_commit = "7438480b2a1bc6371d748e974f7a3647f90c4e8d" - -http_archive( - name = "com_google_googleapis", - strip_prefix = "googleapis-%s" % _googleapis_commit, - urls = [ - "https://github.com/googleapis/googleapis/archive/%s.zip" % _googleapis_commit, - ], -) - -# protobuf -RULES_JVM_EXTERNAL_TAG = "4.5" - -RULES_JVM_EXTERNAL_SHA = "b17d7388feb9bfa7f2fa09031b32707df529f26c91ab9e5d909eb1676badd9a6" - -http_archive( - name = "rules_jvm_external", - sha256 = RULES_JVM_EXTERNAL_SHA, - strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, - url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, -) - -load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") - -rules_jvm_external_deps() - -load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") - -rules_jvm_external_setup() - -load("@com_google_protobuf//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") -load("@rules_jvm_external//:defs.bzl", "maven_install") - -maven_install( - artifacts = PROTOBUF_MAVEN_ARTIFACTS, - repositories = ["https://repo.maven.apache.org/maven2/"], -) - -_gapic_generator_java_version = "2.25.1-SNAPSHOT" # {x-version-update:gapic-generator-java:current} - -maven_install( - artifacts = [ - "com.google.api:gapic-generator-java:" + _gapic_generator_java_version, - ], - fail_on_missing_checksum = False, - repositories = [ - "m2Local", - "https://repo.maven.apache.org/maven2/", - ], -) - -protobuf_deps() - -# Bazel rules. -_rules_gapic_version = "0.5.5" - -http_archive( - name = "rules_gapic", - strip_prefix = "rules_gapic-%s" % _rules_gapic_version, - urls = ["https://github.com/googleapis/rules_gapic/archive/v%s.tar.gz" % _rules_gapic_version], -) - -# Java dependencies. -load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") - -switched_rules_by_language( - name = "com_google_googleapis_imports", - gapic = True, - grpc = True, - java = True, -) - -load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories") - -grpc_java_repositories() - -_disco_to_proto3_converter_commit = "ce8d8732120cdfb5bf4847c3238b5be8acde87e3" - -http_archive( - name = "com_google_disco_to_proto3_converter", - strip_prefix = "disco-to-proto3-converter-%s" % _disco_to_proto3_converter_commit, - urls = ["https://github.com/googleapis/disco-to-proto3-converter/archive/%s.zip" % _disco_to_proto3_converter_commit], -) - -# Showcase -_showcase_version = "0.28.2" - -http_archive( - name = "com_google_gapic_showcase", - strip_prefix = "gapic-showcase-%s" % _showcase_version, - urls = [ - "https://github.com/googleapis/gapic-showcase/archive/refs/tags/v%s.zip" % _showcase_version, - ], -) - -http_archive( - name = "rules_pkg", - sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", - "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", - ], -) - -load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") - -rules_pkg_dependencies() diff --git a/library_generation/test/resources/misc/versions.txt b/library_generation/test/resources/misc/versions.txt deleted file mode 100644 index e4258504e4..0000000000 --- a/library_generation/test/resources/misc/versions.txt +++ /dev/null @@ -1,9 +0,0 @@ -# test versions.txt obtained from sdk-platform-java - -gapic-generator-java:2.25.0:2.25.1-SNAPSHOT -api-common:2.16.0:2.16.1-SNAPSHOT -gax:2.33.0:2.33.1-SNAPSHOT -gax-grpc:2.34.0:2.33.1-SNAPSHOT -gax-httpjson:0.118.0:0.118.1-SNAPSHOT -proto-google-common-protos:2.24.0:2.24.1-SNAPSHOT -grpc-google-common-protos:2.24.0:2.24.1-SNAPSHOT diff --git a/library_generation/test/resources/proto_path_library/proto-test-library/src/main/proto/google/cloud/test/v1/empty.proto b/library_generation/test/resources/proto_path_library/proto-test-library/src/main/proto/google/cloud/test/v1/empty.proto deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/resources/proto_path_library_multiple_protos/proto-1/fake.proto b/library_generation/test/resources/proto_path_library_multiple_protos/proto-1/fake.proto deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/resources/proto_path_library_multiple_protos/proto-2/fake.proto b/library_generation/test/resources/proto_path_library_multiple_protos/proto-2/fake.proto deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/resources/test-config/config_without_api_description.yaml b/library_generation/test/resources/test-config/config_without_api_description.yaml deleted file mode 100644 index 79ff135067..0000000000 --- a/library_generation/test/resources/test-config/config_without_api_description.yaml +++ /dev/null @@ -1,5 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/library_generation/test/resources/test-config/config_without_api_shortname.yaml b/library_generation/test/resources/test-config/config_without_api_shortname.yaml deleted file mode 100644 index ec8206be61..0000000000 --- a/library_generation/test/resources/test-config/config_without_api_shortname.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/library_generation/test/resources/test-config/config_without_gapics_key.yaml b/library_generation/test/resources/test-config/config_without_gapics_key.yaml deleted file mode 100644 index 739a4d9239..0000000000 --- a/library_generation/test/resources/test-config/config_without_gapics_key.yaml +++ /dev/null @@ -1,3 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect diff --git a/library_generation/test/resources/test-config/config_without_gapics_value.yaml b/library_generation/test/resources/test-config/config_without_gapics_value.yaml deleted file mode 100644 index ec49e4a669..0000000000 --- a/library_generation/test/resources/test-config/config_without_gapics_value.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - GAPICs: diff --git a/library_generation/test/resources/test-config/config_without_googleapis.yaml b/library_generation/test/resources/test-config/config_without_googleapis.yaml deleted file mode 100644 index e5a00ca4ee..0000000000 --- a/library_generation/test/resources/test-config/config_without_googleapis.yaml +++ /dev/null @@ -1,8 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/library_generation/test/resources/test-config/config_without_libraries.yaml b/library_generation/test/resources/test-config/config_without_libraries.yaml deleted file mode 100644 index dbbe2ea318..0000000000 --- a/library_generation/test/resources/test-config/config_without_libraries.yaml +++ /dev/null @@ -1 +0,0 @@ -gapic_generator_version: 2.34.0 diff --git a/library_generation/test/resources/test-config/config_without_library_value.yaml b/library_generation/test/resources/test-config/config_without_library_value.yaml deleted file mode 100644 index 174a293000..0000000000 --- a/library_generation/test/resources/test-config/config_without_library_value.yaml +++ /dev/null @@ -1,2 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: diff --git a/library_generation/test/resources/test-config/config_without_name_pretty.yaml b/library_generation/test/resources/test-config/config_without_name_pretty.yaml deleted file mode 100644 index f8612ad9ca..0000000000 --- a/library_generation/test/resources/test-config/config_without_name_pretty.yaml +++ /dev/null @@ -1,6 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - api_description: "allows the Apigee hybrid management" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/library_generation/test/resources/test-config/config_without_product_docs.yaml b/library_generation/test/resources/test-config/config_without_product_docs.yaml deleted file mode 100644 index e3921d2c0d..0000000000 --- a/library_generation/test/resources/test-config/config_without_product_docs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/library_generation/test/resources/test-config/config_without_proto_path.yaml b/library_generation/test/resources/test-config/config_without_proto_path.yaml deleted file mode 100644 index e37b0cef63..0000000000 --- a/library_generation/test/resources/test-config/config_without_proto_path.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - GAPICs: - - random_key: diff --git a/library_generation/test/resources/test-config/config_without_temp_excludes.yaml b/library_generation/test/resources/test-config/config_without_temp_excludes.yaml deleted file mode 100644 index 0d1bb7deea..0000000000 --- a/library_generation/test/resources/test-config/config_without_temp_excludes.yaml +++ /dev/null @@ -1,10 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/library_generation/test/resources/test-config/generation_config.yaml b/library_generation/test/resources/test-config/generation_config.yaml deleted file mode 100644 index 168c8fd9a5..0000000000 --- a/library_generation/test/resources/test-config/generation_config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -gapic_generator_version: 2.34.0 -protoc_version: 25.2 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - codeowner_team: "@googleapis/analytics-dpe" - excluded_poms: proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1 - excluded_dependencies: google-iam-policy - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 diff --git a/library_generation/test/resources/test-config/generation_config_library_modified.yaml b/library_generation/test/resources/test-config/generation_config_library_modified.yaml deleted file mode 100644 index f9ae96693b..0000000000 --- a/library_generation/test/resources/test-config/generation_config_library_modified.yaml +++ /dev/null @@ -1,14 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 diff --git a/library_generation/test/resources/test-config/generation_config_with_duplicate_library_name.yaml b/library_generation/test/resources/test-config/generation_config_with_duplicate_library_name.yaml deleted file mode 100644 index c5613f4308..0000000000 --- a/library_generation/test/resources/test-config/generation_config_with_duplicate_library_name.yaml +++ /dev/null @@ -1,51 +0,0 @@ -gapic_generator_version: 2.34.0 -protoc_version: 25.2 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 -template_excludes: - - ".github/*" - - ".kokoro/*" - - "samples/*" - - "CODE_OF_CONDUCT.md" - - "CONTRIBUTING.md" - - "LICENSE" - - "SECURITY.md" - - "java.header" - - "license-checks.xml" - - "renovate.json" - - ".gitignore" -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - codeowner_team: "@googleapis/analytics-dpe" - excluded_poms: proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1 - excluded_dependencies: google-iam-policy - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - - api_shortname: another-cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - GAPICs: - - proto_path: google/cloud/asset/v1 diff --git a/library_generation/test/resources/test-config/monorepo_baseline.yaml b/library_generation/test/resources/test-config/monorepo_baseline.yaml deleted file mode 100644 index c2c4fd4a3b..0000000000 --- a/library_generation/test/resources/test-config/monorepo_baseline.yaml +++ /dev/null @@ -1,30 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: preview - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - api_shortname: cloudbuild - name_pretty: Cloud Build - product_documentation: https://cloud.google.com/cloud-build/ - api_description: lets you build software quickly across all languages. Get complete - control over defining custom workflows for building, testing, and deploying across - multiple environments such as VMs, serverless, Kubernetes, or Firebase. - release_level: stable - distribution_name: com.google.cloud:google-cloud-build - issue_tracker: https://issuetracker.google.com/savedsearches/5226584 - GAPICs: - - proto_path: google/devtools/cloudbuild/v1 - - proto_path: google/devtools/cloudbuild/v2 diff --git a/library_generation/test/resources/test-config/monorepo_current.yaml b/library_generation/test/resources/test-config/monorepo_current.yaml deleted file mode 100644 index 3ee2c8be2c..0000000000 --- a/library_generation/test/resources/test-config/monorepo_current.yaml +++ /dev/null @@ -1,30 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: stable - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - api_shortname: cloudbuild - name_pretty: Cloud Build - product_documentation: https://cloud.google.com/cloud-build/ - api_description: lets you build software quickly across all languages. Get complete - control over defining custom workflows for building, testing, and deploying across - multiple environments such as VMs, serverless, Kubernetes, or Firebase. - release_level: stable - distribution_name: com.google.cloud:google-cloud-build - issue_tracker: https://issuetracker.google.com/savedsearches/5226584 - GAPICs: - - proto_path: google/devtools/cloudbuild/v1 - - proto_path: google/devtools/cloudbuild/v2 diff --git a/library_generation/test/resources/test-config/monorepo_with_common_protos.yaml b/library_generation/test/resources/test-config/monorepo_with_common_protos.yaml deleted file mode 100644 index 6d4c94444a..0000000000 --- a/library_generation/test/resources/test-config/monorepo_with_common_protos.yaml +++ /dev/null @@ -1,43 +0,0 @@ -googleapis_commitish: 6a474b31c53cc1797710206824a17b364a835d2d -gapic_generator_version: 2.34.0 -# the libraries are ordered with respect to library name, which is -# java-{library.library_name} or java-{library.api-shortname} when -# library.library_name is not defined. -libraries: -- api_shortname: common-protos - name_pretty: Common Protos - product_documentation: https://github.com/googleapis/api-common-protos - api_description: Protobuf classes for Google's common protos. - release_level: stable - client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history - distribution_name: com.google.api.grpc:proto-google-common-protos - excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" - excluded_poms: "proto-google-common-protos-bom,proto-google-common-protos" - library_type: OTHER - GAPICs: - - proto_path: google/api - - proto_path: google/apps/card/v1 - - proto_path: google/cloud - - proto_path: google/cloud/audit - - proto_path: google/cloud/location - - proto_path: google/geo/type - - proto_path: google/logging/type - - proto_path: google/longrunning - - proto_path: google/rpc - - proto_path: google/rpc/context - - proto_path: google/shopping/type - - proto_path: google/type -- api_shortname: iam - name_pretty: IAM - product_documentation: https://cloud.google.com/iam - api_description: Manages access control for Google Cloud Platform resources - release_level: stable - client_documentation: https://cloud.google.com/java/docs/reference/proto-google-iam-v1/latest/overview - distribution_name: com.google.api.grpc:proto-google-iam-v1 - excluded_dependencies: "grpc-google-iam-v1" - excluded_poms: "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1" - library_type: OTHER - GAPICs: - - proto_path: google/iam/v1 - - proto_path: google/iam/v2 - - proto_path: google/iam/v2beta \ No newline at end of file diff --git a/library_generation/test/resources/test-config/monorepo_without_common_protos.yaml b/library_generation/test/resources/test-config/monorepo_without_common_protos.yaml deleted file mode 100644 index ca21ccfb01..0000000000 --- a/library_generation/test/resources/test-config/monorepo_without_common_protos.yaml +++ /dev/null @@ -1,33 +0,0 @@ -gapic_generator_version: 2.34.0 -protoc_version: 25.2 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - api_shortname: cloudbuild - name_pretty: Cloud Build - product_documentation: https://cloud.google.com/cloud-build/ - api_description: lets you build software quickly across all languages. Get complete - control over defining custom workflows for building, testing, and deploying across - multiple environments such as VMs, serverless, Kubernetes, or Firebase. - release_level: stable - distribution_name: com.google.cloud:google-cloud-build - issue_tracker: https://issuetracker.google.com/savedsearches/5226584 - GAPICs: - - proto_path: google/devtools/cloudbuild/v1 - - proto_path: google/devtools/cloudbuild/v2 diff --git a/library_generation/test/resources/test-monorepo/.github/.OwlBot.lock.yaml b/library_generation/test/resources/test-monorepo/.github/.OwlBot.lock.yaml deleted file mode 100644 index 77200af4c9..0000000000 --- a/library_generation/test/resources/test-monorepo/.github/.OwlBot.lock.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -docker: - image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:fb7584f6adb3847ac480ed49a4bfe1463965026b2919a1be270e3174f3ce1191 - # created: 2023-01-20T00:00:00.000000000Z diff --git a/library_generation/test/resources/test-monorepo/test-service/.repo-metadata.json b/library_generation/test/resources/test-monorepo/test-service/.repo-metadata.json deleted file mode 100644 index d5b5078213..0000000000 --- a/library_generation/test/resources/test-monorepo/test-service/.repo-metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "api_shortname": "cloudasset", - "name_pretty": "Cloud Asset Inventory", - "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview", - "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", - "release_level": "stable", - "transport": "grpc", - "requires_billing": true, - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-asset", - "distribution_name": "com.google.cloud:google-cloud-asset", - "api_id": "cloudasset.googleapis.com", - "library_type": "GAPIC_AUTO" -} diff --git a/library_generation/test/resources/test-owlbot/fixtures/java_templates/release-please-update/.github/release-please.yml b/library_generation/test/resources/test-owlbot/fixtures/java_templates/release-please-update/.github/release-please.yml deleted file mode 100644 index 807174cce1..0000000000 --- a/library_generation/test/resources/test-owlbot/fixtures/java_templates/release-please-update/.github/release-please.yml +++ /dev/null @@ -1,6 +0,0 @@ -releaseType: java-yoshi -bumpMinorPreMajor: true -branches: -- releaseType: java-lts - bumpMinorPreMajor: true - branch: 1.127.12-sp \ No newline at end of file diff --git a/library_generation/test/resources/test-owlbot/fixtures/java_templates/release-please-update/.repo-metadata.json b/library_generation/test/resources/test-owlbot/fixtures/java_templates/release-please-update/.repo-metadata.json deleted file mode 100644 index 840e69ca7a..0000000000 --- a/library_generation/test/resources/test-owlbot/fixtures/java_templates/release-please-update/.repo-metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "api_shortname": "cloudasset", - "name_pretty": "Cloud Asset Inventory", - "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", - "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html", - "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", - "release_level": "stable", - "transport": "grpc", - "requires_billing": true, - "language": "java", - "repo": "googleapis/java-asset", - "repo_short": "java-asset", - "distribution_name": "com.google.cloud:google-cloud-asset", - "library_type": "GAPIC_AUTO", - "api_id": "cloudasset.googleapis.com" -} \ No newline at end of file diff --git a/library_generation/test/resources/test-owlbot/fixtures/java_templates/render-readme/.repo-metadata.json b/library_generation/test/resources/test-owlbot/fixtures/java_templates/render-readme/.repo-metadata.json deleted file mode 100644 index 840e69ca7a..0000000000 --- a/library_generation/test/resources/test-owlbot/fixtures/java_templates/render-readme/.repo-metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "api_shortname": "cloudasset", - "name_pretty": "Cloud Asset Inventory", - "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", - "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html", - "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", - "release_level": "stable", - "transport": "grpc", - "requires_billing": true, - "language": "java", - "repo": "googleapis/java-asset", - "repo_short": "java-asset", - "distribution_name": "com.google.cloud:google-cloud-asset", - "library_type": "GAPIC_AUTO", - "api_id": "cloudasset.googleapis.com" -} \ No newline at end of file diff --git a/library_generation/test/resources/test-owlbot/fixtures/java_templates/standard/.repo-metadata.json b/library_generation/test/resources/test-owlbot/fixtures/java_templates/standard/.repo-metadata.json deleted file mode 100644 index 840e69ca7a..0000000000 --- a/library_generation/test/resources/test-owlbot/fixtures/java_templates/standard/.repo-metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "api_shortname": "cloudasset", - "name_pretty": "Cloud Asset Inventory", - "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", - "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html", - "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", - "release_level": "stable", - "transport": "grpc", - "requires_billing": true, - "language": "java", - "repo": "googleapis/java-asset", - "repo_short": "java-asset", - "distribution_name": "com.google.cloud:google-cloud-asset", - "library_type": "GAPIC_AUTO", - "api_id": "cloudasset.googleapis.com" -} \ No newline at end of file diff --git a/library_generation/test/resources/test-owlbot/java-admanager/.repo-metadata.json b/library_generation/test/resources/test-owlbot/java-admanager/.repo-metadata.json deleted file mode 100644 index 1c680ddfea..0000000000 --- a/library_generation/test/resources/test-owlbot/java-admanager/.repo-metadata.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "api_shortname": "admanager", - "name_pretty": "Google Ad Manager API", - "product_documentation": "https://developers.google.com/ad-manager/api/beta", - "api_description": "The Ad Manager API enables an app to integrate with Google Ad Manager. You can read Ad Manager data and run reports using the API.", - "client_documentation": "https://cloud.google.com/java/docs/reference/ad-manager/latest/overview", - "release_level": "preview", - "transport": "http", - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-admanager", - "distribution_name": "com.google.api-ads:ad-manager", - "api_id": "admanager.googleapis.com", - "library_type": "GAPIC_AUTO", - "requires_billing": true -} \ No newline at end of file diff --git a/library_generation/test/resources/test-owlbot/java-admanager/ad-manager-bom/pom-golden.xml b/library_generation/test/resources/test-owlbot/java-admanager/ad-manager-bom/pom-golden.xml deleted file mode 100644 index 2adae69c60..0000000000 --- a/library_generation/test/resources/test-owlbot/java-admanager/ad-manager-bom/pom-golden.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - 4.0.0 - com.google.api-ads - ad-manager-bom - 0.2.0-SNAPSHOT - pom - - com.google.cloud - google-cloud-pom-parent - 1.37.0-SNAPSHOT - ../../google-cloud-pom-parent/pom.xml - - - Google Google Ad Manager API BOM - - BOM for Google Ad Manager API - - - - true - - - - - - com.google.api-ads - ad-manager - 0.2.0-SNAPSHOT - - - com.google.api-ads.api.grpc - proto-ad-manager-v1 - 0.2.0-SNAPSHOT - - - - diff --git a/library_generation/test/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml b/library_generation/test/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml deleted file mode 100644 index 4c8a8a8343..0000000000 --- a/library_generation/test/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - 4.0.0 - com.google.api-ads - ad-manager - 0.2.0-SNAPSHOT - jar - Google Google Ad Manager API - Google Ad Manager API The Ad Manager API enables an app to integrate with Google Ad Manager. You can read Ad Manager data and run reports using the API. - - com.google.api-ads - ad-manager-parent - 0.2.0-SNAPSHOT - - - ad-manager - - - - io.grpc - grpc-api - - - io.grpc - grpc-stub - - - io.grpc - grpc-protobuf - - - com.google.api - api-common - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - - com.google.api-ads.api.grpc - proto-ad-manager-v1 - - - com.google.guava - guava - - - com.google.api - gax - - - com.google.api - gax-grpc - - - com.google.api - gax-httpjson - - - com.google.api.grpc - grpc-google-common-protos - test - - - com.google.api.grpc - proto-google-iam-v1 - - - com.google.api.grpc - grpc-google-iam-v1 - test - - - org.threeten - threetenbp - - - - - junit - junit - test - - - - - com.google.api - gax - testlib - test - - - com.google.api - gax-grpc - testlib - test - - - com.google.api - gax-httpjson - testlib - test - - - diff --git a/library_generation/test/resources/test-owlbot/java-admanager/pom-golden.xml b/library_generation/test/resources/test-owlbot/java-admanager/pom-golden.xml deleted file mode 100644 index 669d294ae7..0000000000 --- a/library_generation/test/resources/test-owlbot/java-admanager/pom-golden.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - 4.0.0 - com.google.api-ads - ad-manager-parent - pom - 0.2.0-SNAPSHOT - Google Google Ad Manager API Parent - - Java idiomatic client for Google Cloud Platform services. - - - - com.google.cloud - google-cloud-jar-parent - 1.37.0-SNAPSHOT - ../google-cloud-jar-parent/pom.xml - - - - UTF-8 - UTF-8 - github - ad-manager-parent - - - - - - com.google.api-ads - ad-manager - 0.2.0-SNAPSHOT - - - com.google.api-ads.api.grpc - proto-ad-manager-v1 - 0.2.0-SNAPSHOT - - - - - - - ad-manager - proto-ad-manager-v1 - ad-manager-bom - - - diff --git a/library_generation/test/resources/test-owlbot/java-admanager/proto-ad-manager-v1/pom-golden.xml b/library_generation/test/resources/test-owlbot/java-admanager/proto-ad-manager-v1/pom-golden.xml deleted file mode 100644 index eef103e533..0000000000 --- a/library_generation/test/resources/test-owlbot/java-admanager/proto-ad-manager-v1/pom-golden.xml +++ /dev/null @@ -1,37 +0,0 @@ - - 4.0.0 - com.google.api-ads.api.grpc - proto-ad-manager-v1 - 0.2.0-SNAPSHOT - proto-ad-manager-v1 - Proto library for ad-manager - - com.google.api-ads - ad-manager-parent - 0.2.0-SNAPSHOT - - - - com.google.protobuf - protobuf-java - - - com.google.api.grpc - proto-google-common-protos - - - com.google.api.grpc - proto-google-iam-v1 - - - com.google.api - api-common - - - com.google.guava - guava - - - diff --git a/library_generation/test/resources/test-owlbot/java-admanager/versions.txt b/library_generation/test/resources/test-owlbot/java-admanager/versions.txt deleted file mode 100644 index 3f073728f3..0000000000 --- a/library_generation/test/resources/test-owlbot/java-admanager/versions.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Format: -# module:released-version:current-version - -google-cloud-java:1.36.0:1.37.0-SNAPSHOT -ad-manager:0.1.0:0.2.0-SNAPSHOT -proto-ad-manager-v1:0.1.0:0.2.0-SNAPSHOT diff --git a/library_generation/test/resources/test-owlbot/testdata/FooGrpcGolden.java b/library_generation/test/resources/test-owlbot/testdata/FooGrpcGolden.java deleted file mode 100644 index 64f7bb90e2..0000000000 --- a/library_generation/test/resources/test-owlbot/testdata/FooGrpcGolden.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package foo; - -// This class is intentionally missing a license header - -class FooGrpc { - -} diff --git a/library_generation/test/resources/test-owlbot/testdata/FooProtoGolden.java b/library_generation/test/resources/test-owlbot/testdata/FooProtoGolden.java deleted file mode 100644 index 75bfe578be..0000000000 --- a/library_generation/test/resources/test-owlbot/testdata/FooProtoGolden.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: foo/foo_proto.proto - -package foo; - -// This class is intentionally missing a license header - -class FooProto { - -} \ No newline at end of file diff --git a/library_generation/test/resources/test-owlbot/testdata/README-golden.md b/library_generation/test/resources/test-owlbot/testdata/README-golden.md deleted file mode 100644 index bb4eb039f5..0000000000 --- a/library_generation/test/resources/test-owlbot/testdata/README-golden.md +++ /dev/null @@ -1,202 +0,0 @@ -# Google Cloud Asset Inventory Client for Java - -Java idiomatic client for [Cloud Asset Inventory][product-docs]. - -[![Maven][maven-version-image]][maven-version-link] -![Stability][stability-image] - -- [Product Documentation][product-docs] -- [Client Library Documentation][javadocs] - - -:bus: In October 2022, this library has moved to -[google-cloud-java/java-asset]( -https://github.com/googleapis/google-cloud-java/tree/main/java-asset). -This repository will be archived in the future. -Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). -The Maven artifact coordinates (`com.google.cloud:google-cloud-asset`) remain the same. - -## Quickstart - - -If you are using Maven, add this to your pom.xml file: - - -```xml - - com.google.cloud - google-cloud-asset - 1.2.3 - -``` - -If you are using Gradle without BOM, add this to your dependencies: - -```Groovy -implementation 'com.google.cloud:google-cloud-asset:1.2.3' -``` - -If you are using SBT, add this to your dependencies: - -```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-asset" % "1.2.3" -``` - -## Authentication - -See the [Authentication][authentication] section in the base directory's README. - -## Authorization - -The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired Cloud Asset Inventory APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the Cloud Asset Inventory API calls. - -## Getting Started - -### Prerequisites - -You will need a [Google Cloud Platform Console][developer-console] project with the Cloud Asset Inventory [API enabled][enable-api]. -You will need to [enable billing][enable-billing] to use Google Cloud Asset Inventory. -[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by -[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: -`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. - -### Installation and setup - -You'll need to obtain the `google-cloud-asset` library. See the [Quickstart](#quickstart) section -to add `google-cloud-asset` as a dependency in your code. - -## About Cloud Asset Inventory - - -[Cloud Asset Inventory][product-docs] provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe. - -See the [Cloud Asset Inventory client library docs][javadocs] to learn how to -use this Cloud Asset Inventory Client Library. - - - - - - -## Troubleshooting - -To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. - -## Transport - -Cloud Asset Inventory uses gRPC for the transport layer. - -## Supported Java Versions - -Java 8 or above is required for using this client. - -Google's Java client libraries, -[Google Cloud Client Libraries][cloudlibs] -and -[Google Cloud API Libraries][apilibs], -follow the -[Oracle Java SE support roadmap][oracle] -(see the Oracle Java SE Product Releases section). - -### For new development - -In general, new feature development occurs with support for the lowest Java -LTS version covered by Oracle's Premier Support (which typically lasts 5 years -from initial General Availability). If the minimum required JVM for a given -library is changed, it is accompanied by a [semver][semver] major release. - -Java 11 and (in September 2021) Java 17 are the best choices for new -development. - -### Keeping production systems current - -Google tests its client libraries with all current LTS versions covered by -Oracle's Extended Support (which typically lasts 8 years from initial -General Availability). - -#### Legacy support - -Google's client libraries support legacy versions of Java runtimes with long -term stable libraries that don't receive feature updates on a best efforts basis -as it may not be possible to backport all patches. - -Google provides updates on a best efforts basis to apps that continue to use -Java 7, though apps might need to upgrade to current versions of the library -that supports their JVM. - -#### Where to find specific information - -The latest versions and the supported Java versions are identified on -the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` -and on [google-cloud-java][g-c-j]. - -## Versioning - - -This library follows [Semantic Versioning](http://semver.org/). - - - -## Contributing - - -Contributions to this library are always welcome and highly encouraged. - -See [CONTRIBUTING][contributing] for more information how to get started. - -Please note that this project is released with a Contributor Code of Conduct. By participating in -this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more -information. - - -## License - -Apache 2.0 - See [LICENSE][license] for more information. - -## CI Status - -Java Version | Status ------------- | ------ -Java 8 | [![Kokoro CI][kokoro-badge-image-2]][kokoro-badge-link-2] -Java 8 OSX | [![Kokoro CI][kokoro-badge-image-3]][kokoro-badge-link-3] -Java 8 Windows | [![Kokoro CI][kokoro-badge-image-4]][kokoro-badge-link-4] -Java 11 | [![Kokoro CI][kokoro-badge-image-5]][kokoro-badge-link-5] - -Java is a registered trademark of Oracle and/or its affiliates. - -[product-docs]: https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview -[javadocs]: https://googleapis.dev/java/google-cloud-asset/latest/index.html -[kokoro-badge-image-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java7.svg -[kokoro-badge-link-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java7.html -[kokoro-badge-image-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8.svg -[kokoro-badge-link-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8.html -[kokoro-badge-image-3]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8-osx.svg -[kokoro-badge-link-3]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8-osx.html -[kokoro-badge-image-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8-win.svg -[kokoro-badge-link-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8-win.html -[kokoro-badge-image-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java11.svg -[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java11.html -[stability-image]: https://img.shields.io/badge/stability-stable-green -[maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-asset.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-asset/1.2.3 -[authentication]: https://github.com/googleapis/google-cloud-java#authentication -[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes -[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles -[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy -[developer-console]: https://console.developers.google.com/ -[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects -[cloud-cli]: https://cloud.google.com/cli -[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md -[contributing]: https://github.com/googleapis/java-asset/blob/main/CONTRIBUTING.md -[code-of-conduct]: https://github.com/googleapis/java-asset/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct -[license]: https://github.com/googleapis/java-asset/blob/main/LICENSE -[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing -[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid=cloudasset.googleapis.com -[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM -[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png - -[semver]: https://semver.org/ -[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained -[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries -[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html -[g-c-j]: http://github.com/googleapis/google-cloud-java diff --git a/library_generation/test/resources/test-owlbot/testdata/SampleClass.java b/library_generation/test/resources/test-owlbot/testdata/SampleClass.java deleted file mode 100644 index 849548b9da..0000000000 --- a/library_generation/test/resources/test-owlbot/testdata/SampleClass.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dlp; - -class ExampleClass { - public static void foo() { - System.out.println("bar"); - } - - public static class InnerClass { - public void asdf() { - System.out.println("qwer"); - } - } -} diff --git a/library_generation/test/resources/test-owlbot/testdata/SampleClassGolden.java b/library_generation/test/resources/test-owlbot/testdata/SampleClassGolden.java deleted file mode 100644 index dbe1fe9d18..0000000000 --- a/library_generation/test/resources/test-owlbot/testdata/SampleClassGolden.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dlp; - -class ExampleClass { - - public static class InnerClass { - } -} diff --git a/library_generation/test/resources/test-owlbot/testdata/SampleCopyMethodGolden.java b/library_generation/test/resources/test-owlbot/testdata/SampleCopyMethodGolden.java deleted file mode 100644 index 02ce855bc9..0000000000 --- a/library_generation/test/resources/test-owlbot/testdata/SampleCopyMethodGolden.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dlp; - -class ExampleClass { - public static void foo() { - System.out.println("bar"); - } - - public static void foobar() { - System.out.println("bar"); - } - - public static class InnerClass { - public void asdf() { - System.out.println("qwer"); - } - - public void xyz() { - System.out.println("qwer"); - } - } -} diff --git a/library_generation/test/resources/test-owlbot/testdata/SampleDeprecateClass.java b/library_generation/test/resources/test-owlbot/testdata/SampleDeprecateClass.java deleted file mode 100644 index d2ae795106..0000000000 --- a/library_generation/test/resources/test-owlbot/testdata/SampleDeprecateClass.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -class ExampleClass { - public void cat(String bar) { - for (int i = 0; i < 3; i++) { - System.out.println("this is a test " + bar); - } - } - - @Beta - @Generated() - public void foo(String bar) { - for (int i = 0; i < 3; i++) { - System.out.println("this is a test " + bar); - } - } - - /** - * This is an existing comment. - */ - public void bar(String bar) { - for (int i = 0; i < 3; i++) { - System.out.println("this is a test " + bar); - } - } -} diff --git a/library_generation/test/resources/test-owlbot/testdata/SampleDeprecateMethodGolden.java b/library_generation/test/resources/test-owlbot/testdata/SampleDeprecateMethodGolden.java deleted file mode 100644 index a5107d9ce6..0000000000 --- a/library_generation/test/resources/test-owlbot/testdata/SampleDeprecateMethodGolden.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -class ExampleClass { - /** - * @deprecated sample has the same functionality as foobar. - */ - @Deprecated - public void cat(String bar) { - for (int i = 0; i < 3; i++) { - System.out.println("this is a test " + bar); - } - } - - /** - * @deprecated This method will be removed in the next major version. - * Use {@link #sample()} instead - */ - @Beta - @Generated() - @Deprecated - public void foo(String bar) { - for (int i = 0; i < 3; i++) { - System.out.println("this is a test " + bar); - } - } - - /** - * This is an existing comment. - * @deprecated This method will be removed in the next major version. - * Use {@link #sample()} instead - */ - @Deprecated - public void bar(String bar) { - for (int i = 0; i < 3; i++) { - System.out.println("this is a test " + bar); - } - } -} diff --git a/library_generation/test/resources/test-owlbot/testdata/src/foo/FooGrpc.java b/library_generation/test/resources/test-owlbot/testdata/src/foo/FooGrpc.java deleted file mode 100644 index be6abc0d64..0000000000 --- a/library_generation/test/resources/test-owlbot/testdata/src/foo/FooGrpc.java +++ /dev/null @@ -1,7 +0,0 @@ -package foo; - -// This class is intentionally missing a license header - -class FooGrpc { - -} diff --git a/library_generation/test/resources/test-owlbot/testdata/src/foo/FooProto.java b/library_generation/test/resources/test-owlbot/testdata/src/foo/FooProto.java deleted file mode 100644 index c3aee1cffe..0000000000 --- a/library_generation/test/resources/test-owlbot/testdata/src/foo/FooProto.java +++ /dev/null @@ -1,10 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: foo/foo_proto.proto - -package foo; - -// This class is intentionally missing a license header - -class FooProto { - -} \ No newline at end of file diff --git a/library_generation/test/resources/test_generate_release_note/empty_gen_config.yaml b/library_generation/test/resources/test_generate_release_note/empty_gen_config.yaml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml b/library_generation/test/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml deleted file mode 100644 index 304ee9b892..0000000000 --- a/library_generation/test/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - 4.0.0 - com.google.cloud - gapic-libraries-bom - pom - 1.29.0-SNAPSHOT - Google Cloud Java BOM - - BOM for the libraries in google-cloud-java repository. Users should not - depend on this artifact explicitly because this BOM is an implementation - detail of the Libraries BOM. - - - - google-cloud-pom-parent - com.google.cloud - 1.29.0-SNAPSHOT - ../google-cloud-pom-parent/pom.xml - - - - - - com.google.cloud - google-cloud-dns - 2.33.0-SNAPSHOT - - - com.google.cloud - google-cloud-service-control-bom - 1.35.0-SNAPSHOT - pom - import - - - com.google.cloud - google-cloud-tasks-bom - 2.35.0-SNAPSHOT - pom - import - - - - \ No newline at end of file diff --git a/library_generation/test/resources/test_monorepo_postprocessing/java-dns/pom.xml b/library_generation/test/resources/test_monorepo_postprocessing/java-dns/pom.xml deleted file mode 100644 index 28bdaad76b..0000000000 --- a/library_generation/test/resources/test_monorepo_postprocessing/java-dns/pom.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-dns - jar - 2.33.0-SNAPSHOT - Google Cloud DNS Parent - diff --git a/library_generation/test/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml b/library_generation/test/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml deleted file mode 100644 index 483838475d..0000000000 --- a/library_generation/test/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-service-control-bom - 1.35.0-SNAPSHOT - pom - diff --git a/library_generation/test/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml b/library_generation/test/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml deleted file mode 100644 index 3138a26ce7..0000000000 --- a/library_generation/test/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-tasks-bom - 2.35.0-SNAPSHOT - pom - diff --git a/library_generation/test/resources/test_monorepo_postprocessing/pom-golden.xml b/library_generation/test/resources/test_monorepo_postprocessing/pom-golden.xml deleted file mode 100644 index d7c0a1f6a1..0000000000 --- a/library_generation/test/resources/test_monorepo_postprocessing/pom-golden.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - 4.0.0 - google-cloud-java - com.google.cloud - 0.201.0 - pom - - - true - - - - gapic-libraries-bom - google-cloud-jar-parent - google-cloud-pom-parent - java-dns - java-service-control - java-tasks - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.1.3 - - true - - - - - - - - release-staging-repository - - - - !gpg.executable - - - - - sonatype-nexus-snapshots - https://google.oss.sonatype.org/content/repositories/snapshots - - - sonatype-nexus-staging - https://google.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.7.0 - true - - sonatype-nexus-staging - https://google.oss.sonatype.org/ - false - - - - - - - release-non-google-oss-sonatype - - - - org.sonatype.plugins - nexus-staging-maven-plugin - - ossrh - https://oss.sonatype.org/ - - - - - - - \ No newline at end of file diff --git a/library_generation/test/resources/test_monorepo_postprocessing/versions.txt b/library_generation/test/resources/test_monorepo_postprocessing/versions.txt deleted file mode 100644 index 6a537f4a39..0000000000 --- a/library_generation/test/resources/test_monorepo_postprocessing/versions.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Format: -# module:released-version:current-version - -google-cloud-java:1.28.0:1.29.0-SNAPSHOT diff --git a/library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt b/library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt b/library_generation/test/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt b/library_generation/test/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt b/library_generation/test/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt b/library_generation/test/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/test_utilities.sh b/library_generation/test/test_utilities.sh deleted file mode 100755 index bbcdee1ebc..0000000000 --- a/library_generation/test/test_utilities.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env bash - -set -xeo pipefail -test_utilities_script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -# Utility functions commonly used in test cases. - -# Variables used to generate final result -total_num=0 -succeed_num=0 -failed_num=0 -failed_tests="" - -############# Helper functions, they shouldn't be called outside this file ############# -__test_executed() { - total_num=$((1 + total_num)) -} - -__test_succeed() { - succeed_num=$((1 + succeed_num)) -} - -__test_failed() { - failed_test=$1 - failed_num=$((1 + failed_num)) - failed_tests="${failed_tests} ${failed_test}" -} - - -############# Functions used in test execution. They can only be called once per test ############# - -assertEquals() { - local expected=$1 - local actual=$2 - if [[ "${expected}" == "${actual}" ]]; then - __test_succeed - return - fi - - echo "Error: expected ${expected}, got ${actual} instead." - __test_failed "${ut}" -} - -assertFileOrDirectoryExists() { - local expected_file=$1 - if [ -d "${expected_file}" ]|| [ -f "${expected_file}" ]; then - __test_succeed - return - fi - - echo "Error: ${expected_file} does not exist." - __test_failed "${ut}" -} - -# Clean up generated files, tooling when testing `generate_library.sh`. -cleanup() { - local library_directory=$1 - rm -rf ../"${library_directory}" \ - google/protobuf \ - protobuf-* \ - gapic-generator-java-*.jar \ - gapic-generator-java-pom-parent-*.pom \ - protoc-gen-grpc-*.exe -} - -execute_tests() { - local test_list=("$@") - for ut in "${test_list[@]}"; do - echo "========== Execute ${ut} ==========" - __test_executed - "${ut}" - done - - echo "Test result: ${total_num} tests executed, ${succeed_num} succeed, ${failed_num} failed." - if [[ "${total_num}" == "${succeed_num}" ]]; then - echo "All tests passed." - exit - fi - - echo "Test failed." - echo "Failed test(s): ${failed_tests}." - exit 1 -} - -############# Utility functions used in `generate_library_integration_tests.sh` ############# - -# Obtains a version from a bazel WORKSPACE file -# -# versions look like "_ggj_version="1.2.3" -# It will return 1.2.3 for such example - -# performs a deep structural comparison between the current pom in a git -# folder and the one at HEAD. -# This function is OS-dependent, so it sources the main utilities script to -# perform detection -compare_poms() { - target_dir=$1 - source "${test_utilities_script_dir}/../utilities.sh" - os_architecture=$(detect_os_architecture) - pushd "${target_dir}" &> /dev/null - find . -name 'pom.xml' -exec cp {} {}.new \; - find . -name 'pom.xml' -exec git checkout HEAD -- {} \; - # compare_poms.py exits with non-zero if diffs are found - set -e - result=0 - if [ "${os_architecture}" == "linux-x86_64" ]; then - find . -name 'pom.xml' -print0 | xargs -i -0 python3 "${test_utilities_script_dir}/compare_poms.py" {} {}.new false || result=$? - else - find . -name 'pom.xml' -print0 | xargs -I{} -0 python3 "${test_utilities_script_dir}/compare_poms.py" {} {}.new false || result=$? - fi - popd &> /dev/null # target_dir - echo ${result} -} - -# computes the `destination_path` variable by inspecting the contents of the -# googleapis-gen at $proto_path. -compute_destination_path() { - local proto_path=$1 - local output_folder=$2 - pushd "${output_folder}" &> /dev/null - local destination_path=$(find "googleapis-gen/${proto_path}" -maxdepth 1 -name 'google-*-java' \ - | rev \ - | cut -d'/' -f1 \ - | rev - ) - popd &> /dev/null # output_folder - echo "${destination_path}" -} - diff --git a/library_generation/test/test_utils.py b/library_generation/test/test_utils.py deleted file mode 100644 index 890cd95362..0000000000 --- a/library_generation/test/test_utils.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest -from difflib import unified_diff -from pathlib import Path - -from typing import List - - -class FileComparator(unittest.TestCase): - def compare_files(self, expect: str, actual: str): - with open(expect, "r") as f: - expected_lines = f.readlines() - with open(actual, "r") as f: - actual_lines = f.readlines() - - diff = list(unified_diff(expected_lines, actual_lines)) - self.assertEqual( - first=[], second=diff, msg="Unexpected file contents:\n" + "".join(diff) - ) - - -def cleanup(files: List[str]): - for file in files: - path = Path(file).resolve() - if path.is_file(): - path.unlink() - elif path.is_dir(): - path.rmdir() diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py deleted file mode 100644 index 3970c5c233..0000000000 --- a/library_generation/test/utilities_unit_tests.py +++ /dev/null @@ -1,391 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Unit tests for python scripts -""" -import shutil -import unittest -import os -import io -import contextlib -from pathlib import Path -from library_generation.utils import utilities as util -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.gapic_inputs import GapicInputs -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.library_config import LibraryConfig -from library_generation.test.test_utils import FileComparator -from library_generation.test.test_utils import cleanup - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "resources") -file_comparator = FileComparator() -library_1 = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - gapic_configs=list(), - library_name="bare-metal-solution", - rest_documentation="https://cloud.google.com/bare-metal/docs/reference/rest", - rpc_documentation="https://cloud.google.com/bare-metal/docs/reference/rpc", - recommended_package="com.google.example", - min_java_version=8, -) -library_2 = LibraryConfig( - api_shortname="secretmanager", - name_pretty="Secret Management", - product_documentation="https://cloud.google.com/solutions/secrets-management/", - api_description="allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", - gapic_configs=list(), -) -common_protos = LibraryConfig( - api_shortname="common-protos", - name_pretty="Common Protos", - product_documentation="", - api_description="example description", - gapic_configs=list(), -) -test_library_with_custom_transport = LibraryConfig( - api_shortname="secretmanager", - name_pretty="Secret Management", - product_documentation="https://cloud.google.com/solutions/secrets-management/", - api_description="allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", - gapic_configs=list(), - transport="rest", -) - - -class UtilitiesTest(unittest.TestCase): - """ - Unit tests for utilities.py - """ - - CONFIGURATION_YAML_PATH = os.path.join( - script_dir, - "resources", - "integration", - "google-cloud-java", - "generation_config.yaml", - ) - - def test_create_argument_valid_container_succeeds(self): - container_value = "google/test/v1" - container = GapicConfig(container_value) - argument_key = "proto_path" - result = util.create_argument(argument_key, container) - self.assertEqual([f"--{argument_key}", container_value], result) - - def test_create_argument_empty_container_returns_empty_list(self): - container = dict() - argument_key = "proto_path" - result = util.create_argument(argument_key, container) - self.assertEqual([], result) - - def test_create_argument_none_container_fails(self): - container = None - argument_key = "proto_path" - result = util.create_argument(argument_key, container) - self.assertEqual([], result) - - def test_sh_util_existent_function_succeeds(self): - result = util.sh_util("extract_folder_name path/to/folder_name") - self.assertEqual("folder_name", result) - - def test_sh_util_nonexistent_function_fails(self): - with self.assertRaises(RuntimeError): - result = util.sh_util("nonexistent_function") - - def test_mv_src_files_gapic_main_succeeds(self): - previous_dir = os.getcwd() - os.chdir(f"{resources_dir}/test_mv_src/gapic") - os.environ["folder_name"] = "example" - util.sh_util("mv_src_files gapic main destination") - self.assertTrue( - os.path.isfile("destination/gapic-example/src/main/java/example_main.txt") - ) - shutil.rmtree("destination/gapic-example") - os.chdir(previous_dir) - - def test_mv_src_files_gapic_test_succeeds(self): - previous_dir = os.getcwd() - os.chdir(f"{resources_dir}/test_mv_src/gapic") - os.environ["folder_name"] = "example" - util.sh_util("mv_src_files gapic test destination") - self.assertTrue( - os.path.isfile("destination/gapic-example/src/test/java/example_test.txt") - ) - shutil.rmtree("destination/gapic-example") - os.chdir(previous_dir) - - def test_mv_src_files_proto_main_succeeds(self): - previous_dir = os.getcwd() - os.chdir(f"{resources_dir}/test_mv_src/proto") - os.environ["folder_name"] = "example" - util.sh_util("mv_src_files proto main destination") - self.assertTrue( - os.path.isfile( - "destination/proto-example/src/main/java/example_proto_main.txt" - ) - ) - shutil.rmtree("destination/proto-example") - os.chdir(previous_dir) - - def test_mv_src_files_sample_suffix_io_succeeds(self): - previous_dir = os.getcwd() - os.chdir(f"{resources_dir}/test_mv_src/samples") - util.sh_util("mv_src_files samples main destination_io") - self.assertTrue( - os.path.isfile( - "destination_io/samples/snippets/generated/io/example_io_sample.txt" - ) - ) - shutil.rmtree("destination_io/samples") - os.chdir(previous_dir) - - def test_mv_src_files_sample_suffix_com_succeeds(self): - previous_dir = os.getcwd() - os.chdir(f"{resources_dir}/test_mv_src/samples") - util.sh_util("mv_src_files samples main destination_com") - self.assertTrue( - os.path.isfile( - "destination_com/samples/snippets/generated/com/example_com_sample.txt" - ) - ) - shutil.rmtree("destination_com/samples") - os.chdir(previous_dir) - - def test_eprint_valid_input_succeeds(self): - test_input = "This is some test input" - # create a stdio capture object - stderr_capture = io.StringIO() - # run eprint() with the capture object - with contextlib.redirect_stderr(stderr_capture): - util.eprint(test_input) - result = stderr_capture.getvalue() - # print() appends a `\n` each time it's called - self.assertEqual(test_input + "\n", result) - - def test_generate_postprocessing_prerequisite_files_non_monorepo_success(self): - library_path = self.__setup_postprocessing_prerequisite_files( - combination=1, library_type="GAPIC_COMBO" - ) - - file_comparator.compare_files( - f"{library_path}/.repo-metadata.json", - f"{library_path}/.repo-metadata-non-monorepo-golden.json", - ) - # since this is a single library, we treat this as HW repository, - # meaning that the owlbot yaml will be inside a .github folder - file_comparator.compare_files( - f"{library_path}/.github/.OwlBot-hermetic.yaml", - f"{library_path}/.OwlBot-hermetic-golden.yaml", - ) - file_comparator.compare_files( - f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" - ) - self.__remove_postprocessing_prerequisite_files( - path=library_path, is_monorepo=False - ) - - def test_generate_postprocessing_prerequisite_files_monorepo_success(self): - library_path = self.__setup_postprocessing_prerequisite_files(combination=2) - - file_comparator.compare_files( - f"{library_path}/.repo-metadata.json", - f"{library_path}/.repo-metadata-monorepo-golden.json", - ) - file_comparator.compare_files( - f"{library_path}/.OwlBot-hermetic.yaml", - f"{library_path}/.OwlBot-hermetic-golden.yaml", - ) - file_comparator.compare_files( - f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" - ) - self.__remove_postprocessing_prerequisite_files(path=library_path) - - def test_generate_postprocessing_prerequisite_files_proto_only_repo_success(self): - library_path = self.__setup_postprocessing_prerequisite_files( - combination=3, library_type="OTHER" - ) - - file_comparator.compare_files( - f"{library_path}/.repo-metadata.json", - f"{library_path}/.repo-metadata-proto-only-golden.json", - ) - file_comparator.compare_files( - f"{library_path}/.OwlBot-hermetic.yaml", - f"{library_path}/.OwlBot-hermetic-golden.yaml", - ) - file_comparator.compare_files( - f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" - ) - self.__remove_postprocessing_prerequisite_files(path=library_path) - - def test_generate_postprocessing_prerequisite_files__custom_transport_set_in_config__success( - self, - ): - """ - This test generates files for `test_library_with_custom_transport`, which - has an explicit value for transport declared (http). This is expected to - override the value obtained in BUILD.bazel via gapic_inputs.parse(). For - testing purposes, we test with a default GapicInputs object, whose transport - is set to "grpc". - """ - library_path = self.__setup_postprocessing_prerequisite_files( - combination=2, library=test_library_with_custom_transport - ) - - file_comparator.compare_files( - f"{library_path}/.repo-metadata.json", - f"{library_path}/.repo-metadata-custom-transport-golden.json", - ) - self.__remove_postprocessing_prerequisite_files(path=library_path) - - def test_create__library_invalid_transport__fails( - self, - ): - - with self.assertRaises(ValueError): - test_library_with_invalid_transport = LibraryConfig( - api_shortname="secretmanager", - name_pretty="Secret Management", - product_documentation="https://cloud.google.com/solutions/secrets-management/", - api_description="allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", - gapic_configs=list(), - transport="http", - ) - - def test_prepare_repo_monorepo_success(self): - gen_config = self.__get_a_gen_config(2) - repo_config = util.prepare_repo( - gen_config=gen_config, - library_config=gen_config.libraries, - repo_path=f"{resources_dir}/misc", - ) - self.assertEqual("output", Path(repo_config.output_folder).name) - library_path = sorted([Path(key).name for key in repo_config.libraries]) - self.assertEqual( - ["java-bare-metal-solution", "java-secretmanager"], library_path - ) - - def test_prepare_repo_monorepo_failed(self): - gen_config = self.__get_a_gen_config(2) - self.assertRaises( - FileNotFoundError, - util.prepare_repo, - gen_config, - gen_config.libraries, - f"{resources_dir}/non-exist", - ) - - def test_prepare_repo_split_repo_success(self): - gen_config = self.__get_a_gen_config(1) - repo_config = util.prepare_repo( - gen_config=gen_config, - library_config=gen_config.libraries, - repo_path=f"{resources_dir}/misc", - ) - self.assertEqual("output", Path(repo_config.output_folder).name) - library_path = sorted([Path(key).name for key in repo_config.libraries]) - self.assertEqual(["misc"], library_path) - shutil.rmtree(repo_config.output_folder) - - def test_get_java_generator_location_success(self): - location = util.sh_util("get_gapic_generator_location") - self.assertRegex(location, r"/.library_generation/gapic-generator-java.jar$") - - def test_get_java_formatter_location_success(self): - location = util.sh_util("get_java_formatter_location") - self.assertRegex(location, r"/.library_generation/google-java-format.jar$") - - def __setup_postprocessing_prerequisite_files( - self, - combination: int, - library_type: str = "GAPIC_AUTO", - library: LibraryConfig = library_1, - ) -> str: - library_path = f"{resources_dir}/goldens" - files = [ - f"{library_path}/.repo-metadata.json", - f"{library_path}/.OwlBot-hermetic.yaml", - f"{library_path}/owlbot.py", - ] - cleanup(files) - library.library_type = library_type - config = self.__get_a_gen_config(combination, library_type=library_type) - proto_path = "google/cloud/baremetalsolution/v2" - gapic_inputs = GapicInputs() # defaults to transport=grpc - transport = library.get_transport(gapic_inputs) - util.generate_postprocessing_prerequisite_files( - config=config, - library=library, - proto_path=proto_path, - transport=transport, - library_path=library_path, - ) - return library_path - - @staticmethod - def __get_a_gen_config( - combination: int, library_type: str = "GAPIC_AUTO" - ) -> GenerationConfig: - """ - Returns an object of GenerationConfig with one to three of - LibraryConfig objects. Other attributes are set to empty str. - - :param combination: combination of LibraryConfig objects associated with - the GenerationConfig. Only support 1, 2 or 3. - :return: an object of GenerationConfig - """ - if combination == 1: - libraries = [library_1] - elif combination == 2: - libraries = [library_1, library_2] - else: - libraries = [library_1, common_protos] - - # update libraries with custom configuration (for now, only - # library_type) - for library in libraries: - library.library_type = library_type - if combination == 1: - # treat this as a HW library case to generate a real-life - # repo-metadata - library.extra_versioned_modules = "test-module" - else: - library.extra_versioned_modules = None - - return GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - libraries=libraries, - ) - - @staticmethod - def __remove_postprocessing_prerequisite_files( - path: str, is_monorepo: bool = True - ) -> None: - os.remove(f"{path}/.repo-metadata.json") - os.remove(f"{path}/owlbot.py") - if is_monorepo: - os.remove(f"{path}/.OwlBot-hermetic.yaml") - return - if os.path.isdir(f"{path}/.github"): - shutil.rmtree(f"{path}/.github", ignore_errors=True) - - -if __name__ == "__main__": - unittest.main() diff --git a/library_generation/test/utils/__init__.py b/library_generation/test/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/test/utils/commit_message_formatter_unit_tests.py b/library_generation/test/utils/commit_message_formatter_unit_tests.py deleted file mode 100644 index 16e3fffdfc..0000000000 --- a/library_generation/test/utils/commit_message_formatter_unit_tests.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest -from unittest.mock import patch - -from library_generation.model.config_change import ( - ConfigChange, - ChangeType, - LibraryChange, -) -from library_generation.model.generation_config import GenerationConfig -from library_generation.utils.commit_message_formatter import ( - format_commit_message, - commit_link, - format_repo_level_change, -) -from library_generation.utils.commit_message_formatter import wrap_googleapis_commit -from library_generation.utils.commit_message_formatter import wrap_override_commit - -gen_config = GenerationConfig( - gapic_generator_version="1.2.3", googleapis_commitish="123abc", libraries=[] -) - - -class CommitMessageFormatterTest(unittest.TestCase): - def test_format_commit_message_should_add_library_name_for_conventional_commit( - self, - ): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.message = "feat: a commit message\nPiperOrigin-RevId: 123456" - commit.hexsha = "1234567abcdefg" - commits = {commit: "example_library"} - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "feat: [example_library] a commit message", - "PiperOrigin-RevId: 123456", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - format_commit_message(commits, True), - ) - - def test_format_commit_message_should_add_library_name_for_mutliline_conventional_commit( - self, - ): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.message = "feat: a commit message\nfix: an another commit message\nPiperOrigin-RevId: 123456" - commit.hexsha = "1234567abcdefg" - commits = {commit: "example_library"} - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "feat: [example_library] a commit message", - "fix: [example_library] an another commit message", - "PiperOrigin-RevId: 123456", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - format_commit_message(commits, True), - ) - - def test_format_commit_message_should_not_add_library_name_for_nonconvnentional_commit( - self, - ): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.message = "PiperOrigin-RevId: 123456" - commit.hexsha = "1234567abcdefg" - commits = {commit: "example_library"} - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "PiperOrigin-RevId: 123456", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - format_commit_message(commits, True), - ) - - def test_format_commit_message_should_not_add_library_name_if_not_monorepo(self): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.message = "feat: a commit message\nPiperOrigin-RevId: 123456" - commit.hexsha = "1234567abcdefg" - commits = {commit: "example_library"} - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "feat: a commit message", - "PiperOrigin-RevId: 123456", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - format_commit_message(commits, False), - ) - - def test_format_commit_message_should_not_add_library_name_for_multiline_commit_if_not_monorepo( - self, - ): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.message = "feat: a commit message\nfix: an another commit message\nPiperOrigin-RevId: 123456" - commit.hexsha = "1234567abcdefg" - commits = {commit: "example_library"} - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "feat: a commit message", - "fix: an another commit message", - "PiperOrigin-RevId: 123456", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - format_commit_message(commits, False), - ) - - def test_wrap_nested_commit_success(self): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.hexsha = "1234567abcdefg" - messages = ["a commit message", "another message"] - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "a commit message", - "another message", - "Source Link: [googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - "END_NESTED_COMMIT", - ], - wrap_googleapis_commit(commit, messages), - ) - - def test_wrap_override_commit_success(self): - messages = ["a commit message", "another message"] - self.assertEqual( - [ - "BEGIN_COMMIT_OVERRIDE", - "a commit message", - "another message", - "END_COMMIT_OVERRIDE", - ], - wrap_override_commit(messages), - ) - - def test_commit_link_success(self): - with patch("git.Commit") as mock_commit: - commit = mock_commit.return_value - commit.hexsha = "1234567abcdefg" - self.assertEqual( - "[googleapis/googleapis@1234567](https://github.com/googleapis/googleapis/commit/1234567abcdefg)", - commit_link(commit), - ) - - def test_format_repo_level_change_success(self): - config_change = ConfigChange( - change_to_libraries={ - ChangeType.REPO_LEVEL_CHANGE: [ - LibraryChange( - changed_param="gapic_generator_version", current_value="1.2.3" - ), - LibraryChange( - changed_param="libraries_bom_version", current_value="2.3.4" - ), - LibraryChange( - changed_param="protoc_version", current_value="3.4.5" - ), - ] - }, - baseline_config=gen_config, - current_config=gen_config, - ) - self.assertEqual( - [ - "BEGIN_NESTED_COMMIT", - "fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3", - "END_NESTED_COMMIT", - "BEGIN_NESTED_COMMIT", - "chore: update the libraries_bom version to 2.3.4", - "END_NESTED_COMMIT", - "BEGIN_NESTED_COMMIT", - "chore: update repo-level parameter protoc_version to 3.4.5", - "END_NESTED_COMMIT", - ], - format_repo_level_change(config_change), - ) diff --git a/library_generation/test/utils/generation_config_comparator_unit_tests.py b/library_generation/test/utils/generation_config_comparator_unit_tests.py deleted file mode 100644 index 8d018a7c15..0000000000 --- a/library_generation/test/utils/generation_config_comparator_unit_tests.py +++ /dev/null @@ -1,506 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.library_config import LibraryConfig -from library_generation.utils.generation_config_comparator import ChangeType -from library_generation.utils.generation_config_comparator import compare_config - - -class GenerationConfigComparatorTest(unittest.TestCase): - def setUp(self) -> None: - self.baseline_library = LibraryConfig( - api_shortname="existing_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], - ) - self.current_library = LibraryConfig( - api_shortname="existing_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], - ) - self.baseline_config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - grpc_version="", - protoc_version="", - libraries=[self.baseline_library], - ) - self.current_config = GenerationConfig( - gapic_generator_version="", - googleapis_commitish="", - grpc_version="", - protoc_version="", - libraries=[self.current_library], - ) - - def test_compare_config_not_change(self): - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue(len(result.change_to_libraries) == 0) - - def test_compare_config_googleapis_update(self): - self.baseline_config.googleapis_commitish = ( - "1a45bf7393b52407188c82e63101db7dc9c72026" - ) - self.current_config.googleapis_commitish = ( - "1e6517ef4f949191c9e471857cf5811c8abcab84" - ) - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertEqual({ChangeType.GOOGLEAPIS_COMMIT: []}, result.change_to_libraries) - - def test_compare_config_generator_update(self): - self.baseline_config.gapic_generator_version = "1.2.3" - self.current_config.gapic_generator_version = "1.2.4" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("gapic_generator_version", config_change.changed_param) - self.assertEqual("1.2.4", config_change.current_value) - - def test_compare_config_libraries_bom_update(self): - self.baseline_config.libraries_bom_version = "26.36.0" - self.current_config.libraries_bom_version = "26.37.0" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("libraries_bom_version", config_change.changed_param) - self.assertEqual("26.37.0", config_change.current_value) - - def test_compare_protobuf_update(self): - self.baseline_config.protoc_version = "3.25.2" - self.current_config.protoc_version = "3.27.0" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("protoc_version", config_change.changed_param) - self.assertEqual("3.27.0", config_change.current_value) - - def test_compare_config_grpc_update(self): - self.baseline_config.grpc_version = "1.60.0" - self.current_config.grpc_version = "1.61.0" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("grpc_version", config_change.changed_param) - self.assertEqual("1.61.0", config_change.current_value) - - def test_compare_config_template_excludes_update(self): - self.baseline_config.template_excludes = [".github/*", ".kokoro/*"] - self.current_config.template_excludes = [ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - ] - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] - self.assertEqual("template_excludes", config_change.changed_param) - self.assertEqual( - [ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - ], - config_change.current_value, - ) - - def test_compare_config_library_addition(self): - self.current_config.libraries.append( - LibraryConfig( - api_shortname="new_library", - api_description="", - name_pretty="", - product_documentation="", - gapic_configs=[], - ) - ) - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] - self.assertEqual("new_library", config_change.library_name) - - def test_compare_config_library_removal_does_not_have_repo_or_library_level_change( - self, - ): - self.current_config.libraries = [] - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 0 - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 0 - ) - - def test_compare_config_api_shortname_update_without_library_name(self): - self.current_config.libraries[0].api_shortname = "new_api_shortname" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] - self.assertEqual("new_api_shortname", config_change.library_name) - - def test_compare_config_api_shortname_update_with_library_name_raise_error(self): - self.baseline_config.libraries[0].library_name = "old_library_name" - self.current_config.libraries[0].library_name = "old_library_name" - self.current_config.libraries[0].api_shortname = "new_api_shortname" - self.assertRaisesRegex( - ValueError, - r"api_shortname.*library_name", - compare_config, - self.baseline_config, - self.current_config, - ) - - def test_compare_config_library_name_update(self): - self.current_config.libraries[0].library_name = "new_library_name" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] - self.assertEqual("new_library_name", config_change.library_name) - - def test_compare_config_api_description_update(self): - self.current_config.libraries[0].api_description = "updated description" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("api_description", config_change.changed_param) - self.assertEqual("updated description", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_name_pretty_update(self): - self.current_config.libraries[0].name_pretty = "new name" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("name_pretty", config_change.changed_param) - self.assertEqual("new name", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_product_docs_update(self): - self.current_config.libraries[0].product_documentation = "new docs" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("product_documentation", config_change.changed_param) - self.assertEqual("new docs", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_library_type_update(self): - self.current_config.libraries[0].library_type = "GAPIC_COMBO" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("library_type", config_change.changed_param) - self.assertEqual("GAPIC_COMBO", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_release_level_update(self): - self.current_config.libraries[0].release_level = "STABLE" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("release_level", config_change.changed_param) - self.assertEqual("STABLE", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_api_id_update(self): - self.current_config.libraries[0].api_id = "new_id" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("api_id", config_change.changed_param) - self.assertEqual("new_id", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_api_reference_update(self): - self.current_config.libraries[0].api_reference = "new api_reference" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("api_reference", config_change.changed_param) - self.assertEqual("new api_reference", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_code_owner_team_update(self): - self.current_config.libraries[0].codeowner_team = "new team" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("codeowner_team", config_change.changed_param) - self.assertEqual("new team", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_excluded_deps_update(self): - self.current_config.libraries[0].excluded_dependencies = "group:artifact" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("excluded_dependencies", config_change.changed_param) - self.assertEqual("group:artifact", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_excluded_poms_update(self): - self.current_config.libraries[0].excluded_poms = "pom.xml" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("excluded_poms", config_change.changed_param) - self.assertEqual("pom.xml", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_client_docs_update(self): - self.current_config.libraries[0].client_documentation = "new client docs" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("client_documentation", config_change.changed_param) - self.assertEqual("new client docs", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_distribution_name_update(self): - self.current_config.libraries[0].distribution_name = "new_group:new_artifact" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("distribution_name", config_change.changed_param) - self.assertEqual("new_group:new_artifact", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_group_id_update(self): - self.current_config.libraries[0].group_id = "new_group" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("group_id", config_change.changed_param) - self.assertEqual("new_group", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_issue_tracker_update(self): - self.current_config.libraries[0].issue_tracker = "new issue tracker" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("issue_tracker", config_change.changed_param) - self.assertEqual("new issue tracker", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_rest_docs_update(self): - self.current_config.libraries[0].rest_documentation = "new rest docs" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("rest_documentation", config_change.changed_param) - self.assertEqual("new rest docs", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_rpc_docs_update(self): - self.current_config.libraries[0].rpc_documentation = "new rpc docs" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("rpc_documentation", config_change.changed_param) - self.assertEqual("new rpc docs", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_cloud_api_update(self): - self.current_config.libraries[0].cloud_api = False - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("cloud_api", config_change.changed_param) - self.assertEqual(False, config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_requires_billing_update(self): - self.current_config.libraries[0].requires_billing = False - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("requires_billing", config_change.changed_param) - self.assertEqual(False, config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_extra_versioned_mod_update(self): - self.current_config.libraries[0].extra_versioned_modules = "extra module" - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue( - len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 - ) - config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] - self.assertEqual("extra_versioned_modules", config_change.changed_param) - self.assertEqual("extra module", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) - - def test_compare_config_version_addition(self): - self.current_config.libraries[0].gapic_configs = [ - GapicConfig(proto_path="google/new/library/v1") - ] - result = compare_config( - baseline_config=self.baseline_config, - current_config=self.current_config, - ) - self.assertTrue(len(result.change_to_libraries[ChangeType.GAPIC_ADDITION]) == 1) - config_change = result.change_to_libraries[ChangeType.GAPIC_ADDITION][0] - self.assertEqual("", config_change.changed_param) - self.assertEqual("google/new/library/v1", config_change.current_value) - self.assertEqual("existing_library", config_change.library_name) diff --git a/library_generation/test/utils/monorepo_postprocessor_unit_tests.py b/library_generation/test/utils/monorepo_postprocessor_unit_tests.py deleted file mode 100644 index 5dfafe2086..0000000000 --- a/library_generation/test/utils/monorepo_postprocessor_unit_tests.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest - -from library_generation.test.test_utils import FileComparator -from library_generation.test.test_utils import cleanup -from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") -file_comparator = FileComparator() - - -class MonorepoPostprocessorTest(unittest.TestCase): - def test_monorepo_postprocessing_valid_repository_success(self): - repository_path = f"{resources_dir}/test_monorepo_postprocessing" - versions_file = f"{repository_path}/versions.txt" - files = [ - f"{repository_path}/pom.xml", - f"{repository_path}/gapic-libraries-bom/pom.xml", - ] - cleanup(files) - monorepo_postprocessing( - repository_path=repository_path, versions_file=versions_file - ) - file_comparator.compare_files( - expect=f"{repository_path}/pom-golden.xml", - actual=f"{repository_path}/pom.xml", - ) - file_comparator.compare_files( - expect=f"{repository_path}/gapic-libraries-bom/pom-golden.xml", - actual=f"{repository_path}/gapic-libraries-bom/pom.xml", - ) - cleanup(files) diff --git a/library_generation/test/utils/pom_generator_unit_tests.py b/library_generation/test/utils/pom_generator_unit_tests.py deleted file mode 100644 index 7a829e58aa..0000000000 --- a/library_generation/test/utils/pom_generator_unit_tests.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest - -from library_generation.utils.pom_generator import get_version_from - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") - - -class PomGeneratorTest(unittest.TestCase): - def test_get_version_from_returns_current(self): - versions_file = f"{resources_dir}/misc/versions.txt" - artifact = "gax-grpc" - self.assertEqual("2.33.1-SNAPSHOT", get_version_from(versions_file, artifact)) - - def test_get_version_from_returns_released(self): - versions_file = f"{resources_dir}/misc/versions.txt" - artifact = "gax-grpc" - self.assertEqual("2.34.0", get_version_from(versions_file, artifact, True)) diff --git a/library_generation/test/utils/proto_path_utils_unit_tests.py b/library_generation/test/utils/proto_path_utils_unit_tests.py deleted file mode 100644 index 2e23c2b403..0000000000 --- a/library_generation/test/utils/proto_path_utils_unit_tests.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import unittest -from pathlib import Path -from library_generation.utils.proto_path_utils import ( - find_versioned_proto_path, - remove_version_from, -) - -script_dir = os.path.dirname(os.path.realpath(__file__)) -resources_dir = os.path.join(script_dir, "..", "resources") -test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() - - -class ProtoPathsUtilsTest(unittest.TestCase): - def test_find_versioned_proto_path_nested_version_success(self): - proto_path = "google/cloud/aiplatform/v1/schema/predict/params/image_classification.proto" - expected = "google/cloud/aiplatform/v1" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - - def test_find_versioned_proto_path_success(self): - proto_path = "google/cloud/asset/v1p2beta1/assets.proto" - expected = "google/cloud/asset/v1p2beta1" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - - def test_find_versioned_proto_without_version_return_itself(self): - proto_path = "google/type/color.proto" - expected = "google/type/color.proto" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - - def test_remove_version_from_returns_non_versioned_path(self): - proto_path = "google/cloud/aiplatform/v1" - self.assertEqual("google/cloud/aiplatform", remove_version_from(proto_path)) - - def test_remove_version_from_returns_self(self): - proto_path = "google/cloud/aiplatform" - self.assertEqual("google/cloud/aiplatform", remove_version_from(proto_path)) diff --git a/library_generation/utils/__init__.py b/library_generation/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/utils/commit_message_formatter.py b/library_generation/utils/commit_message_formatter.py deleted file mode 100644 index 5b75db51a0..0000000000 --- a/library_generation/utils/commit_message_formatter.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re -from git import Commit - -from library_generation.model.config_change import ConfigChange, ChangeType -from library_generation.model.generation_config import ( - GAPIC_GENERATOR_VERSION, - LIBRARIES_BOM_VERSION, -) - -PARAM_TO_COMMIT_MESSAGE = { - GAPIC_GENERATOR_VERSION: "fix(deps): update the Java code generator (gapic-generator-java) to", - LIBRARIES_BOM_VERSION: "chore: update the libraries_bom version to", -} - - -def format_commit_message(commits: dict[Commit, str], is_monorepo: bool) -> list[str]: - """ - Format commit messages. Add library_name to conventional commit messages if - is_monorepo is True; otherwise no op. - - :param commits: a mapping from commit to library_name. - :param is_monorepo: whether it's monorepo or not. - :return: formatted commit messages. - """ - all_commits = [] - # please see go/java-client-releasing#conventional-commit-messages - # for conventional commit. - type_regex = re.compile(r"(feat|fix|docs|deps|test|samples|chore)!?:.*") - for commit, library_name in commits.items(): - # a commit message may contain multiple lines, we need to - # add library_name for each line. - messages = [] - for message_line in commit.message.split("\n"): - # add library name to a conventional commit message; - # otherwise no op. - if type_regex.search(message_line): - commit_type, _, summary = message_line.partition(":") - formatted_message = ( - f"{commit_type}: [{library_name}]{str(summary).rstrip()}" - if is_monorepo - else f"{commit_type}:{str(summary).rstrip()}" - ) - messages.append(formatted_message) - else: - messages.append(message_line) - all_commits.extend(wrap_googleapis_commit(commit, messages)) - return all_commits - - -def format_repo_level_change(config_change: ConfigChange) -> list[str]: - """ - Format commit messages regarding repo-level changes. - - :param config_change: - :return: commit messages regarding repo-level changes. - """ - messages = [] - for repo_level_change in config_change.change_to_libraries.get( - ChangeType.REPO_LEVEL_CHANGE, [] - ): - message = f"chore: update repo-level parameter {repo_level_change.changed_param} to {repo_level_change.current_value}" - if repo_level_change.changed_param in PARAM_TO_COMMIT_MESSAGE: - message = f"{PARAM_TO_COMMIT_MESSAGE.get(repo_level_change.changed_param)} {repo_level_change.current_value}" - messages.extend(__wrap_nested_commit([message])) - return messages - - -def wrap_googleapis_commit(commit: Commit, messages: list[str]) -> list[str]: - """ - Wrap message between `BEGIN_NESTED_COMMIT` and `BEGIN_NESTED_COMMIT`. - - :param commit: a GitHub commit. - :param messages: a (multi-line) commit message, one line per item. - :return: wrapped messages. - """ - messages.append(f"Source Link: {commit_link(commit)}") - return __wrap_nested_commit(messages) - - -def wrap_override_commit(messages: list[str]) -> list[str]: - """ - Wrap message between `BEGIN_COMMIT_OVERRIDE` and `END_COMMIT_OVERRIDE`. - - :param messages: a (multi-line) commit message, one line per item. - :return: wrapped messages. - """ - result = ["BEGIN_COMMIT_OVERRIDE"] - result.extend(messages) - result.append("END_COMMIT_OVERRIDE") - return result - - -def commit_link(commit: Commit) -> str: - """ - Create a link to the commit in Markdown format. - - :param commit: a GitHub commit. - :return: a link in Markdown format. - """ - short_sha = commit.hexsha[:7] - return f"[googleapis/googleapis@{short_sha}](https://github.com/googleapis/googleapis/commit/{commit.hexsha})" - - -def __wrap_nested_commit(messages: list[str]) -> list[str]: - """ - Wrap message between `BEGIN_NESTED_COMMIT` and `BEGIN_NESTED_COMMIT`. - - :param messages: a (multi-line) commit message, one line per item. - :return: wrapped messages. - """ - result = ["BEGIN_NESTED_COMMIT"] - result.extend(messages) - result.append("END_NESTED_COMMIT") - return result diff --git a/library_generation/utils/file_render.py b/library_generation/utils/file_render.py deleted file mode 100644 index 5c68445753..0000000000 --- a/library_generation/utils/file_render.py +++ /dev/null @@ -1,24 +0,0 @@ -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -from jinja2 import Environment, FileSystemLoader - -script_dir = os.path.dirname(os.path.realpath(__file__)) -jinja_env = Environment(loader=FileSystemLoader(f"{script_dir}/../templates")) - - -def render(template_name: str, output_name: str, **kwargs): - template = jinja_env.get_template(template_name) - t = template.stream(kwargs) - directory = os.path.dirname(output_name) - if not os.path.isdir(directory): - os.makedirs(directory) - t.dump(str(output_name)) diff --git a/library_generation/utils/generation_config_comparator.py b/library_generation/utils/generation_config_comparator.py deleted file mode 100644 index d96a09f378..0000000000 --- a/library_generation/utils/generation_config_comparator.py +++ /dev/null @@ -1,284 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections import defaultdict -from typing import Any -from typing import Dict -from typing import List -from library_generation.model.config_change import ChangeType -from library_generation.model.config_change import ConfigChange -from library_generation.model.config_change import LibraryChange -from library_generation.model.config_change import HashLibrary -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.library_config import LibraryConfig - - -def compare_config( - baseline_config: GenerationConfig, current_config: GenerationConfig -) -> ConfigChange: - """ - Compare two GenerationConfig object and output a mapping from ConfigChange - to a list of ConfigChange objects. - All libraries in the current configuration will be affected if one of the - repository level parameters is changed. - - :param baseline_config: the baseline GenerationConfig object - :param current_config: the current GenerationConfig object - :return: a ConfigChange objects. - """ - diff = defaultdict(list[LibraryChange]) - # Exclude `libraries` because an empty library list (e.g., by library - # removal) will cause this parameter appears in the sorted param list, - # which leads to unequal list of parameters. - excluded_params = {"libraries"} - baseline_params = __convert_params_to_sorted_list( - obj=baseline_config, excluded_params=excluded_params - ) - current_params = __convert_params_to_sorted_list( - obj=current_config, excluded_params=excluded_params - ) - - for baseline_param, current_param in zip(baseline_params, current_params): - if baseline_param == current_param: - continue - if baseline_param[0] == "googleapis_commitish": - diff[ChangeType.GOOGLEAPIS_COMMIT] = [] - else: - config_change = LibraryChange( - changed_param=current_param[0], - current_value=current_param[1], - ) - diff[ChangeType.REPO_LEVEL_CHANGE].append(config_change) - - __compare_libraries( - diff=diff, - baseline_library_configs=baseline_config.libraries, - current_library_configs=current_config.libraries, - ) - return ConfigChange( - change_to_libraries=diff, - baseline_config=baseline_config, - current_config=current_config, - ) - - -def __compare_libraries( - diff: Dict[ChangeType, list[LibraryChange]], - baseline_library_configs: List[LibraryConfig], - current_library_configs: List[LibraryConfig], -) -> None: - """ - Compare two lists of LibraryConfig and put the difference into a - given Dict. - - :param diff: a mapping from ConfigChange to a list of ConfigChange objects. - :param baseline_library_configs: a list of LibraryConfig object. - :param current_library_configs: a list of LibraryConfig object. - """ - baseline_libraries = __convert_to_hashed_library_dict(baseline_library_configs) - current_libraries = __convert_to_hashed_library_dict(current_library_configs) - changed_libraries = [] - # 1st round comparison. - for library_name, hash_library in baseline_libraries.items(): - # 1. find any library removed from baseline_libraries. - # a library is removed from baseline_libraries if the library_name - # is not in current_libraries. - # please see the reason of comment out these lines of code in the - # comment of ChangeType.LIBRARIES_REMOVAL. - # if library_name not in current_libraries: - # config_change = ConfigChange( - # changed_param="", current_value="", library_name=library_name - # ) - # diff[ChangeType.LIBRARIES_REMOVAL].append(config_change) - - # 2. find any library that exists in both configs but at least one - # parameter is changed, which means the hash value is different. - if ( - library_name in current_libraries - and hash_library.hash_value != current_libraries[library_name].hash_value - ): - changed_libraries.append(library_name) - # 2nd round comparison. - for library_name in current_libraries: - # find any library added to current_libraries. - # a library is added to current_libraries if the library_name - # is not in baseline_libraries. - if library_name not in baseline_libraries: - config_change = LibraryChange( - changed_param="", current_value="", library_name=library_name - ) - diff[ChangeType.LIBRARIES_ADDITION].append(config_change) - # 3rd round comparison. - __compare_changed_libraries( - diff=diff, - baseline_libraries=baseline_libraries, - current_libraries=current_libraries, - changed_libraries=changed_libraries, - ) - - -def __convert_to_hashed_library_dict( - libraries: List[LibraryConfig], -) -> Dict[str, HashLibrary]: - """ - Convert a list of LibraryConfig objects to a Dict. - For each library object, the key is the library_name of the object, the - value is a HashLibrary object. - - :param libraries: a list of LibraryConfig object. - :return: a mapping from library_name to HashLibrary object. - """ - return { - library.get_library_name(): HashLibrary(hash(library), library) - for library in libraries - } - - -def __compare_changed_libraries( - diff: Dict[ChangeType, list[LibraryChange]], - baseline_libraries: Dict[str, HashLibrary], - current_libraries: Dict[str, HashLibrary], - changed_libraries: List[str], -) -> None: - """ - Compare each library with the same library_name to find what parameters are - changed. - - :param diff: a mapping from ConfigChange to a list of ConfigChange objects. - :param baseline_libraries: a mapping from library_name to HashLibrary - object. - :param current_libraries: a mapping from library_name to HashLibrary object. - :param changed_libraries: a list of library_name of changed libraries. - :raise ValueError: if api_shortname of a library is changed but library_name - remains the same. - """ - for library_name in changed_libraries: - baseline_library = baseline_libraries[library_name].library - current_library = current_libraries[library_name].library - baseline_params = __convert_params_to_sorted_list(baseline_library) - current_params = __convert_params_to_sorted_list(current_library) - for baseline_param, current_param in zip(baseline_params, current_params): - if baseline_param == current_param: - continue - if baseline_param[0] == "api_shortname": - raise ValueError( - f"{library_name}: api_shortname must not change when library_name remains the same." - ) - else: - config_change = LibraryChange( - changed_param=current_param[0], - current_value=current_param[1], - library_name=library_name, - ) - diff[ChangeType.LIBRARY_LEVEL_CHANGE].append(config_change) - - # compare gapic_configs - baseline_gapic_configs = baseline_library.gapic_configs - current_gapic_configs = current_library.gapic_configs - __compare_gapic_configs( - diff=diff, - library_name=library_name, - baseline_gapic_configs=baseline_gapic_configs, - current_gapic_configs=current_gapic_configs, - ) - - -def __compare_gapic_configs( - diff: Dict[ChangeType, list[LibraryChange]], - library_name: str, - baseline_gapic_configs: List[GapicConfig], - current_gapic_configs: List[GapicConfig], -) -> None: - baseline_proto_paths = {config.proto_path for config in baseline_gapic_configs} - current_proto_paths = {config.proto_path for config in current_gapic_configs} - # 1st round of comparison, find any versioned proto_path is removed - # from baseline gapic configs. - # please see the reason of comment out these lines of code in the - # comment of ChangeType.GAPIC_REMOVAL. - # for proto_path in baseline_proto_paths: - # if proto_path in current_proto_paths: - # continue - # config_change = ConfigChange( - # changed_param="", current_value=proto_path, library_name=library_name - # ) - # diff[ChangeType.GAPIC_REMOVAL].append(config_change) - - # 2nd round of comparison, find any versioned proto_path is added - # to current gapic configs. - for proto_path in current_proto_paths: - if proto_path in baseline_proto_paths: - continue - config_change = LibraryChange( - changed_param="", current_value=proto_path, library_name=library_name - ) - diff[ChangeType.GAPIC_ADDITION].append(config_change) - - -def __convert_params_to_sorted_list(obj: Any, excluded_params=None) -> List[tuple]: - """ - Convert the parameter and its value of a given object to a sorted list of - tuples. - - Only the following types of parameters will be considered: - - - str - - bool - - list[str] - - None - - Note that built-in params, e.g., __str__, and methods will be skipped. - - :param obj: an object. - :param excluded_params: excluded params. - :return: a sorted list of tuples. - """ - if excluded_params is None: - excluded_params = set() - param_and_values = [] - for param, value in vars(obj).items(): - if ( - param in excluded_params - or param.startswith("__") # skip built-in params - or callable(getattr(obj, param)) # skip methods - # skip if the type of param is not one of the following types - # 1. str - # 2. bool - # 3. list[str] - # 4. None - or not ( - isinstance(getattr(obj, param), str) - or isinstance(getattr(obj, param), bool) - or __is_list_of_str(obj=obj, param=param) - or getattr(obj, param) is None - ) - ): - continue - param_and_values.append((param, value)) - return sorted(param_and_values) - - -def __is_list_of_str(obj: Any, param: str) -> bool: - """ - Returns True if the type of param of a given object is a list of str; False - otherwise. - - This method is a workaround of https://bugs.python.org/issue28339. - """ - value = getattr(obj, param) - if not isinstance(value, list): - return False - for v in value: - if not isinstance(v, str): - return False - return True diff --git a/library_generation/utils/monorepo_postprocessor.py b/library_generation/utils/monorepo_postprocessor.py deleted file mode 100644 index cd522b83b5..0000000000 --- a/library_generation/utils/monorepo_postprocessor.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from library_generation.utils.pom_generator import generate_gapic_bom -from library_generation.utils.pom_generator import generate_root_pom - - -def monorepo_postprocessing( - repository_path: str, - versions_file: str, -) -> None: - """ - Perform repository level post-processing - :param repository_path: the path of the repository - :param versions_file: the versions_txt contains version of modules - :return: None - """ - generate_root_pom(repository_path=repository_path) - generate_gapic_bom(repository_path=repository_path, versions_file=versions_file) diff --git a/library_generation/utils/pom_generator.py b/library_generation/utils/pom_generator.py deleted file mode 100644 index 8312d4c5a4..0000000000 --- a/library_generation/utils/pom_generator.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from pathlib import Path - -from lxml import etree -from typing import List - -from library_generation.model.bom_config import BomConfig -from library_generation.utils.file_render import render - -project_tag = "{http://maven.apache.org/POM/4.0.0}" -group_id_tag = "groupId" -artifact_tag = "artifactId" -version_tag = "version" - - -def generate_root_pom(repository_path: str) -> None: - print("Regenerating root pom.xml") - modules = __search_for_java_modules(repository_path) - render( - template_name="root-pom.xml.j2", - output_name=f"{repository_path}/pom.xml", - modules=modules, - ) - - -def generate_gapic_bom(repository_path: str, versions_file: str) -> None: - print("Regenerating gapic-libraries-bom") - bom_configs = __search_for_bom_artifact(repository_path) - monorepo_version = get_version_from( - versions_file=versions_file, - artifact_id="google-cloud-java", - ) - render( - template_name="gapic-libraries-bom.xml.j2", - output_name=f"{repository_path}/gapic-libraries-bom/pom.xml", - monorepo_version=monorepo_version, - bom_configs=bom_configs, - ) - - -def get_version_from( - versions_file: str, artifact_id: str, is_released: bool = False -) -> str: - """ - Get version of a given artifact from versions.txt - :param versions_file: the path of version.txt - :param artifact_id: the artifact id - :param is_released: whether returns the released or current version - :return: the version of the artifact - """ - index = 1 if is_released else 2 - with open(versions_file, "r") as f: - for line in f.readlines(): - if artifact_id in line: - return line.split(":")[index].strip() - - -def __search_for_java_modules( - repository_path: str, -) -> List[str]: - repo = Path(repository_path).resolve() - modules = [] - for sub_dir in repo.iterdir(): - if sub_dir.is_dir() and sub_dir.name.startswith("java-"): - modules.append(sub_dir.name) - return sorted(modules) - - -def __search_for_bom_artifact( - repository_path: str, -) -> List[BomConfig]: - repo = Path(repository_path).resolve() - module_exclusions = ["gapic-libraries-bom"] - group_id_inclusions = [ - "com.google.cloud", - "com.google.analytics", - "com.google.area120", - ] - bom_configs = [] - for module in repo.iterdir(): - if module.is_file() or module.name in module_exclusions: - continue - for sub_module in module.iterdir(): - if sub_module.is_dir() and sub_module.name.endswith("-bom"): - root = etree.parse(f"{sub_module}/pom.xml").getroot() - group_id = root.find(f"{project_tag}{group_id_tag}").text - if group_id not in group_id_inclusions: - continue - artifact_id = root.find(f"{project_tag}{artifact_tag}").text - version = root.find(f"{project_tag}{version_tag}").text - index = artifact_id.rfind("-") - version_annotation = artifact_id[:index] - bom_configs.append( - BomConfig( - group_id=group_id, - artifact_id=artifact_id, - version=version, - version_annotation=version_annotation, - ) - ) - # handle edge case: java-grafeas - bom_configs += __handle_special_bom( - repository_path=repository_path, - module="java-grafeas", - group_id="io.grafeas", - artifact_id="grafeas", - ) - # handle edge case: java-dns - bom_configs += __handle_special_bom( - repository_path=repository_path, - module="java-dns", - group_id="com.google.cloud", - artifact_id="google-cloud-dns", - ) - # handle edge case: java-notification - bom_configs += __handle_special_bom( - repository_path=repository_path, - module="java-notification", - group_id="com.google.cloud", - artifact_id="google-cloud-notification", - ) - - return sorted(bom_configs) - - -def __handle_special_bom( - repository_path: str, - module: str, - group_id: str, - artifact_id: str, -) -> List[BomConfig]: - pom = f"{repository_path}/{module}/pom.xml" - if not Path(pom).exists(): - return [] - root = etree.parse(pom).getroot() - version = root.find(f"{project_tag}{version_tag}").text - return [ - BomConfig( - group_id=group_id, - artifact_id=artifact_id, - version=version, - version_annotation=artifact_id, - is_import=False, - ) - ] diff --git a/library_generation/utils/proto_path_utils.py b/library_generation/utils/proto_path_utils.py deleted file mode 100644 index d2ae25f602..0000000000 --- a/library_generation/utils/proto_path_utils.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re - - -def remove_version_from(proto_path: str) -> str: - """ - Remove the version of a proto_path - :param proto_path: versioned proto_path - :return: the proto_path without version - """ - version_pattern = "^v[1-9]" - index = proto_path.rfind("/") - version = proto_path[index + 1 :] - if re.match(version_pattern, version): - return proto_path[:index] - return proto_path - - -def find_versioned_proto_path(proto_path: str) -> str: - """ - Returns a versioned proto_path from a given proto_path; or proto_path itself - if it doesn't contain a versioned proto_path. - :param proto_path: a proto file path - :return: the versioned proto_path - """ - version_regex = re.compile(r"^v[1-9].*") - directories = proto_path.split("/") - for directory in directories: - result = version_regex.search(directory) - if result: - version = result[0] - idx = proto_path.find(version) - return proto_path[:idx] + version - return proto_path diff --git a/library_generation/utils/utilities.py b/library_generation/utils/utilities.py deleted file mode 100755 index 894a06ec02..0000000000 --- a/library_generation/utils/utilities.py +++ /dev/null @@ -1,310 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import sys -import subprocess -import os -import shutil -from pathlib import Path -from typing import Any - -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.library_config import LibraryConfig -from typing import List -from library_generation.model.repo_config import RepoConfig -from library_generation.utils.file_render import render -from library_generation.utils.proto_path_utils import remove_version_from - -script_dir = os.path.dirname(os.path.realpath(__file__)) -SDK_PLATFORM_JAVA = "googleapis/sdk-platform-java" - - -def create_argument(arg_key: str, arg_container: object) -> List[str]: - """ - Generates a list of two elements [argument, value], or returns - an empty array if arg_val is None - """ - arg_val = getattr(arg_container, arg_key, None) - if arg_val is not None: - return [f"--{arg_key}", f"{arg_val}"] - return [] - - -def run_process_and_print_output( - arguments: List[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs -) -> Any: - """ - Runs a process with the given "arguments" list and prints its output. - If the process fails, then the whole program exits - :param arguments: list of strings or string containing the arguments - :param job_name: optional job name to be printed - :param exit_on_fail: True if the main program should exit when the - subprocess exits with non-zero. Used for testing. - :param kwargs: passed to the underlying subprocess.run() call - """ - # split "arguments" if passed as a single string - if isinstance(arguments, str): - arguments = arguments.split(" ") - if "stderr" not in kwargs: - kwargs["stderr"] = subprocess.STDOUT - proc_info = subprocess.run(arguments, stdout=subprocess.PIPE, **kwargs) - print(proc_info.stdout.decode(), end="", flush=True) - print( - f"{job_name} {'finished successfully' if proc_info.returncode == 0 else 'failed'}" - ) - if exit_on_fail and proc_info.returncode != 0: - sys.exit(1) - return proc_info - - -def run_process_and_get_output_string( - arguments: List[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs -) -> Any: - """ - Wrapper of run_process_and_print_output() that returns the merged - stdout and stderr in a single string without its trailing newline char. - """ - return run_process_and_print_output( - arguments, job_name, exit_on_fail, stderr=subprocess.PIPE, **kwargs - ).stdout.decode()[:-1] - - -def sh_util(statement: str, **kwargs) -> str: - """ - Calls a function defined in library_generation/utils/utilities.sh - """ - if "stdout" not in kwargs: - kwargs["stdout"] = subprocess.PIPE - if "stderr" not in kwargs: - kwargs["stderr"] = subprocess.PIPE - output = "" - with subprocess.Popen( - [ - "bash", - "-exc", - f"source {script_dir}/utilities.sh && {statement}", - ], - **kwargs, - ) as proc: - print("command stderr:") - for line in proc.stderr: - print(line.decode(), end="", flush=True) - print("command stdout:") - for line in proc.stdout: - print(line.decode(), end="", flush=True) - output += line.decode() - proc.wait() - if proc.returncode != 0: - raise RuntimeError( - f"function {statement} failed with exit code {proc.returncode}" - ) - # captured stdout may contain a newline at the end, we remove it - if len(output) > 0 and output[-1] == "\n": - output = output[:-1] - return output - - -def eprint(*args, **kwargs): - """ - prints to stderr - """ - print(*args, file=sys.stderr, **kwargs) - - -def prepare_repo( - gen_config: GenerationConfig, - library_config: List[LibraryConfig], - repo_path: str, - language: str = "java", -) -> RepoConfig: - """ - Gather information of the generated repository. - - :param gen_config: a GenerationConfig object representing a parsed - configuration yaml - :param library_config: a LibraryConfig object contained inside config, - passed here for convenience and to prevent all libraries to be processed - :param repo_path: the path to which the generated repository goes - :param language: programming language of the library - :return: a RepoConfig object contained repository information - :raise FileNotFoundError if there's no versions.txt in repo_path - :raise ValueError if two libraries have the same library_name - """ - output_folder = sh_util("get_output_folder") - print(f"output_folder: {output_folder}") - os.makedirs(output_folder, exist_ok=True) - libraries = {} - for library in library_config: - library_name = f"{language}-{library.get_library_name()}" - library_path = ( - f"{repo_path}/{library_name}" - if gen_config.is_monorepo() - else f"{repo_path}" - ) - # use absolute path because docker requires absolute path - # in volume name. - absolute_library_path = str(Path(library_path).resolve()) - libraries[absolute_library_path] = library - # remove existing .repo-metadata.json - json_name = ".repo-metadata.json" - if os.path.exists(f"{absolute_library_path}/{json_name}"): - os.remove(f"{absolute_library_path}/{json_name}") - versions_file = f"{repo_path}/versions.txt" - if not Path(versions_file).exists(): - raise FileNotFoundError(f"{versions_file} is not found.") - - return RepoConfig( - output_folder=output_folder, - libraries=libraries, - versions_file=str(Path(versions_file).resolve()), - ) - - -def generate_postprocessing_prerequisite_files( - config: GenerationConfig, - library: LibraryConfig, - proto_path: str, - transport: str, - library_path: str, - language: str = "java", -) -> None: - """ - Generates the postprocessing prerequisite files for a library. - - :param config: a GenerationConfig object representing a parsed configuration - yaml - :param library: the library configuration - :param proto_path: the path from the root of googleapis to the location of the service - protos. If the path contains a version, it will be removed - :param transport: transport supported by the library - :param library_path: the path to which the generated file goes - :param language: programming language of the library - :return: None - """ - library_name = library.get_library_name() - artifact_id = library.get_artifact_id() - if config.contains_common_protos(): - repo = SDK_PLATFORM_JAVA - elif config.is_monorepo(): - repo = "googleapis/google-cloud-java" - else: - repo = f"googleapis/{language}-{library_name}" - api_id = ( - library.api_id if library.api_id else f"{library.api_shortname}.googleapis.com" - ) - client_documentation = ( - library.client_documentation - if library.client_documentation - else f"https://cloud.google.com/{language}/docs/reference/{artifact_id}/latest/overview" - ) - - # The mapping is needed because transport in .repo-metadata.json - # is one of grpc, http and both, - if transport == "grpc": - converted_transport = "grpc" - elif transport == "rest": - converted_transport = "http" - else: - converted_transport = "both" - - repo_metadata = { - "api_shortname": library.api_shortname, - "name_pretty": library.name_pretty, - "product_documentation": library.product_documentation, - "api_description": library.api_description, - "client_documentation": client_documentation, - "release_level": library.release_level, - "transport": converted_transport, - "language": language, - "repo": repo, - "repo_short": f"{language}-{library_name}", - "distribution_name": library.get_maven_coordinate(), - "api_id": api_id, - "library_type": library.library_type, - "requires_billing": library.requires_billing, - } - - # this removal is for java-common-protos and java-iam in - # sdk-platform-java - if repo == SDK_PLATFORM_JAVA: - repo_metadata.pop("api_id") - - if library.api_reference: - repo_metadata["api_reference"] = library.api_reference - if library.codeowner_team: - repo_metadata["codeowner_team"] = library.codeowner_team - if library.excluded_dependencies: - repo_metadata["excluded_dependencies"] = library.excluded_dependencies - if library.excluded_poms: - repo_metadata["excluded_poms"] = library.excluded_poms - if library.issue_tracker: - repo_metadata["issue_tracker"] = library.issue_tracker - if library.rest_documentation: - repo_metadata["rest_documentation"] = library.rest_documentation - if library.rpc_documentation: - repo_metadata["rpc_documentation"] = library.rpc_documentation - if library.extra_versioned_modules: - repo_metadata["extra_versioned_modules"] = library.extra_versioned_modules - if library.recommended_package: - repo_metadata["recommended_package"] = library.recommended_package - if library.min_java_version: - repo_metadata["min_java_version"] = library.min_java_version - - # generate .repo-meta.json - json_file = ".repo-metadata.json" - # .repo-metadata.json is removed before generating the first version of - # a library. This check ensures no duplicated generation. - if not os.path.exists(f"{library_path}/{json_file}"): - with open(f"{library_path}/{json_file}", "w") as fp: - json.dump(repo_metadata, fp, indent=2) - - # generate .OwlBot-hermetic.yaml - owlbot_yaml_file = ".OwlBot-hermetic.yaml" - path_to_owlbot_yaml_file = ( - f"{library_path}/{owlbot_yaml_file}" - if config.is_monorepo() - else f"{library_path}/.github/{owlbot_yaml_file}" - ) - if not os.path.exists(path_to_owlbot_yaml_file): - render( - template_name="owlbot.yaml.monorepo.j2", - output_name=path_to_owlbot_yaml_file, - artifact_id=artifact_id, - proto_path=remove_version_from(proto_path), - module_name=repo_metadata["repo_short"], - api_shortname=library.api_shortname, - ) - - # generate owlbot.py - py_file = "owlbot.py" - template_excludes = [ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "SECURITY.md", - "java.header", - "license-checks.xml", - "renovate.json", - ".gitignore", - ] - if not os.path.exists(f"{library_path}/{py_file}"): - render( - template_name="owlbot.py.j2", - output_name=f"{library_path}/{py_file}", - should_include_templates=True, - template_excludes=template_excludes, - ) diff --git a/library_generation/utils/utilities.sh b/library_generation/utils/utilities.sh deleted file mode 100755 index a6112070a3..0000000000 --- a/library_generation/utils/utilities.sh +++ /dev/null @@ -1,391 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail -utilities_script_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") -# The $HOME variable is always set in the OS env as per POSIX specification. -GAPIC_GENERATOR_LOCATION="${HOME}/.library_generation/gapic-generator-java.jar" -JAVA_FORMATTER_LOCATION="${HOME}/.library_generation/google-java-format.jar" - -# Utility functions used in `generate_library.sh` and showcase generation. -extract_folder_name() { - local destination_path=$1 - local folder_name=${destination_path##*/} - echo "${folder_name}" -} - -remove_empty_files() { - local category=$1 - local destination_path=$2 - local file_num - find "${destination_path}/${category}-${folder_name}/src/main/java" -type f -size 0 | while read -r f; do rm -f "${f}"; done - # remove the directory if the directory has no files. - file_num=$(find "${destination_path}/${category}-${folder_name}" -type f | wc -l | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - if [[ "${file_num}" == 0 ]]; then - rm -rf "${destination_path}/${category}-${folder_name}" - fi - - if [ -d "${destination_path}/${category}-${folder_name}/src/main/java/samples" ]; then - mv "${destination_path}/${category}-${folder_name}/src/main/java/samples" "${destination_path}/${category}-${folder_name}" - fi -} - -# Move generated files to folders in destination_path. -mv_src_files() { - local category=$1 # one of gapic, proto, samples - local type=$2 # one of main, test - local destination_path=$3 - if [ "${category}" == "samples" ]; then - src_suffix="samples/snippets/generated/src/main/java" - folder_suffix="samples/snippets/generated" - mkdir -p "${destination_path}/${folder_suffix}" - cp -r "${destination_path}/java_gapic_srcjar/${src_suffix}"/* "${destination_path}/${folder_suffix}" - elif [ "${category}" == "proto" ]; then - src_suffix="${category}/src/${type}/java" - folder_suffix="${category}-${folder_name}/src/${type}" - else - src_suffix="src/${type}" - folder_suffix="${category}-${folder_name}/src" - fi - - if [ "${category}" == "samples" ]; then - return - fi - - mkdir -p "${destination_path}/${folder_suffix}" - cp -r "${destination_path}/java_gapic_srcjar/${src_suffix}" "${destination_path}/${folder_suffix}" - rm -r -f "${destination_path}/${folder_suffix}/java/META-INF" -} - -# unzip jar file -unzip_src_files() { - local category=$1 - local destination_path=$2 - local jar_file=java_${category}.jar - mkdir -p "${destination_path}/${category}-${folder_name}/src/main/java" - unzip -q -o "${destination_path}/${jar_file}" -d "${destination_path}/${category}-${folder_name}/src/main/java" - rm -r -f "${destination_path}/${category}-${folder_name}/src/main/java/META-INF" -} - -# get gapic options from .yaml and .json files from proto_path. -get_gapic_opts() { - local transport=$1 - local rest_numeric_enums=$2 - local gapic_yaml=$3 - local service_config=$4 - local service_yaml=$5 - if [ "${rest_numeric_enums}" == "true" ]; then - rest_numeric_enums="rest-numeric-enums" - else - rest_numeric_enums="" - fi - # If any of the gapic options is empty (default value), try to search for - # it in proto_path. - if [[ "${gapic_yaml}" == "" ]]; then - gapic_yaml=$(find "${proto_path}" -type f -name "*gapic.yaml") - fi - - if [[ "${service_config}" == "" ]]; then - service_config=$(find "${proto_path}" -type f -name "*service_config.json") - fi - - if [[ "${service_yaml}" == "" ]]; then - service_yaml=$(find "${proto_path}" -maxdepth 1 -type f \( -name "*.yaml" ! -name "*gapic*.yaml" \)) - fi - echo "transport=${transport},${rest_numeric_enums},grpc-service-config=${service_config},gapic-config=${gapic_yaml},api-service-config=${service_yaml}" -} - -remove_grpc_version() { - local destination_path=$1 - find "${destination_path}" -type f -name "*Grpc.java" -exec \ - sed -i.bak 's/value = \"by gRPC proto compiler.*/value = \"by gRPC proto compiler\",/g' {} \; -exec rm {}.bak \; -} - -# This function returns the version of the grpc plugin to generate the libraries. If -# DOCKER_GRPC_VERSION is set, this will be the version. Otherwise, the script -# will exit since this is a necessary env var -get_grpc_version() { - local grpc_version - if [[ -n "${DOCKER_GRPC_VERSION}" ]]; then - >&2 echo "Using grpc version baked into the container: ${DOCKER_GRPC_VERSION}" - echo "${DOCKER_GRPC_VERSION}" - return - else - >&2 echo "Cannot infer grpc version because DOCKER_GRPC_VERSION is not set" - exit 1 - fi -} - -# This function returns the version of protoc to generate the libraries. If -# DOCKER_PROTOC_VERSION is set, this will be the version. Otherwise, the script -# will exit since this is a necessary env var -get_protoc_version() { - local protoc_version - if [[ -n "${DOCKER_PROTOC_VERSION}" ]]; then - >&2 echo "Using protoc version baked into the container: ${DOCKER_PROTOC_VERSION}" - echo "${DOCKER_PROTOC_VERSION}" - return - else - >&2 echo "Cannot infer protoc version because DOCKER_PROTOC_VERSION is not set" - exit 1 - fi -} - -# Given the versions of the gapic generator, protoc and the protoc-grpc plugin, -# this function will download each one of the tools and create the environment -# variables "protoc_path" and "grpc_path" which are expected upstream. Note that -# if the specified versions of protoc and grpc match DOCKER_PROTOC_VERSION and -# DOCKER_GRPC_VERSION respectively, this function will instead set "protoc_path" -# and "grpc_path" to DOCKER_PROTOC_PATH and DOCKER_GRPC_PATH respectively (no -# download), since the docker image will have downloaded these tools beforehand. -# -# For the case of generator and formatter, no env var will be exported for the -# upstream flow. -# Instead, the jar must be located in the well-known location -# (${HOME}/.library_generation/). -# More information in `library_generation/DEVELOPMENT.md`. -download_tools() { - local protoc_version=$1 - local grpc_version=$2 - local os_architecture=$3 - pushd "${output_folder}" - - # the variable protoc_path is used in generate_library.sh. It is explicitly - # exported to make clear that it is used outside this utilities file. - if [[ "${DOCKER_PROTOC_VERSION}" == "${protoc_version}" ]]; then - # if the specified protoc_version matches the one baked in the docker - # container, we just point protoc_path to its location. - export protoc_path="${DOCKER_PROTOC_LOCATION}/protoc-${protoc_version}/bin" - else - export protoc_path=$(download_protoc "${protoc_version}" "${os_architecture}") - fi - - # similar case with grpc - if [[ "${DOCKER_GRPC_VERSION}" == "${grpc_version}" ]]; then - # if the specified grpc_version matches the one baked in the docker - # container, we just point grpc_path to its location. - export grpc_path="${DOCKER_GRPC_LOCATION}" - else - export grpc_path=$(download_grpc_plugin "${grpc_version}" "${os_architecture}") - fi - - # Here we check whether required tools is stored in the expected location. - # The docker image will prepare jar files in this location. - # This check is meant to ensure integrity of the downstream workflow. - error_if_not_exists "${GAPIC_GENERATOR_LOCATION}" - error_if_not_exists "${JAVA_FORMATTER_LOCATION}" - popd -} - -download_protoc() { - local protoc_version=$1 - local os_architecture=$2 - - local protoc_path="${output_folder}/protoc-${protoc_version}/bin" - - if [ ! -d "${protoc_path}" ]; then - # pull proto files and protoc from protobuf repository as maven central - # doesn't have proto files - download_from \ - "https://github.com/protocolbuffers/protobuf/releases/download/v${protoc_version}/protoc-${protoc_version}-${os_architecture}.zip" \ - "protoc-${protoc_version}.zip" \ - "GitHub" - unzip -o -q "protoc-${protoc_version}.zip" -d "protoc-${protoc_version}" - cp -r "protoc-${protoc_version}/include/google" . - rm "protoc-${protoc_version}.zip" - fi - echo "${protoc_path}" - -} - -download_grpc_plugin() { - local grpc_version=$1 - local os_architecture=$2 - grpc_filename="protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" - if [ ! -f "${grpc_filename}" ]; then - # download protoc-gen-grpc-java plugin from Google maven central mirror. - download_from \ - "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/${grpc_version}/${grpc_filename}" \ - "${grpc_filename}" - chmod +x "${grpc_filename}" - fi - echo "$(pwd)/${grpc_filename}" -} - -download_from() { - local url=$1 - local save_as=$2 - local repo=$3 - # fail-fast, 30 seconds at most, retry 2 times - curl -LJ -o "${save_as}" --fail -m 30 --retry 2 "$url" || download_fail "${save_as}" "${repo}" -} - -# copies the specified file in $1 to $2 -# will return "true" if the copy was successful -copy_from() { - local local_repo=$1 - local save_as=$2 - copy_successful=$(cp "${local_repo}" "${save_as}" && echo "true" || echo "false") - echo "${copy_successful}" -} - -download_fail() { - local artifact=$1 - local repo=${2:-"maven central mirror"} - >&2 echo "Fail to download ${artifact} from ${repo} repository. Please install ${artifact} first if you want to use a non-published artifact." - exit 1 -} - -# gets the output folder where all sources and dependencies will be located. -get_output_folder() { - if [[ $(basename $(pwd)) != "output" ]]; then - echo "$(pwd)/output" - else - echo $(pwd) - fi -} - -detect_os_architecture() { - local os_type - local os_architecture - os_type=$(uname -sm) - case "${os_type}" in - *"Linux x86_64"*) - os_architecture="linux-x86_64" - ;; - *"Darwin x86_64"*) - os_architecture="osx-x86_64" - ;; - *) - os_architecture="osx-aarch_64" - ;; - esac - echo "${os_architecture}" -} - - -# copies $1 as a folder as $2 only if $1 exists -copy_directory_if_exists() { - local base_folder=$1 - local folder_prefix=$2 - local destination_folder=$3 - if [ ! -d "${base_folder}" ]; then - return - fi - pushd "${base_folder}" - if [[ $(find . -maxdepth 1 -type d -name "${folder_prefix}*" | wc -l ) -gt 0 ]]; then - cp -r ${base_folder}/${folder_prefix}* "${destination_folder}" - fi - popd # base_folder -} - -# computes proto_path from a given folder of GAPIC sources -# It will inspect the proto library to compute the path -get_proto_path_from_preprocessed_sources() { - set -e - local sources=$1 - pushd "${sources}" > /dev/null - local proto_library=$(find . -maxdepth 1 -type d -name 'proto-*' | sed 's/\.\///') - local found_libraries=$(echo "${proto_library}" | wc -l) - if [[ -z ${proto_library} ]]; then - echo "no proto libraries found in the supplied sources path" - exit 1 - elif [ ${found_libraries} -gt 1 ]; then - echo "more than one proto library found in the supplied sources path" - echo "cannot decide for a service version" - exit 1 - fi - pushd "$(pwd)/${proto_library}/src/main/proto" > /dev/null - local result=$(find . -type f -name '*.proto' | head -n 1 | xargs dirname | sed 's/\.\///') - popd > /dev/null # proto_library - popd > /dev/null # sources - echo "${result}" -} - -# for a pre-processed library stored in $preprocessed_sources_path, a folder -# tree is built on $target_folder so it looks like a googleapis-gen folder and -# is therefore consumable by OwlBot CLI -build_owlbot_cli_source_folder() { - local postprocessing_target=$1 - local target_folder=$2 - local preprocessed_sources_path=$3 - local proto_path=$4 - if [[ -z "${proto_path}" ]]; then - proto_path=$(get_proto_path_from_preprocessed_sources "${preprocessed_sources_path}") - fi - owlbot_staging_folder="${postprocessing_target}/owl-bot-staging" - mkdir -p "${owlbot_staging_folder}" - - # By default (thanks to generation templates), .OwlBot-hermetic.yaml `deep-copy` section - # references a wildcard pattern matching a folder - # ending with `-java` at the leaf of proto_path. We then use a generated-java - # folder that will be picked up by copy-code - mkdir -p "${target_folder}/${proto_path}/generated-java" - copy_directory_if_exists "${preprocessed_sources_path}" "proto" \ - "${target_folder}/${proto_path}/generated-java/proto-google-cloud-library" - copy_directory_if_exists "${preprocessed_sources_path}" "grpc" \ - "${target_folder}/${proto_path}/generated-java/grpc-google-cloud-library" - copy_directory_if_exists "${preprocessed_sources_path}" "gapic" \ - "${target_folder}/${proto_path}/generated-java/gapic-google-cloud-library" - copy_directory_if_exists "${preprocessed_sources_path}" "samples" \ - "${target_folder}/${proto_path}/generated-java/samples" - pushd "${target_folder}" - # create an empty commit so owl-bot-copy can process this as a repo - # (it cannot process non-git-repositories) - git init - git commit --allow-empty -m 'empty commit' - popd # target_folder -} - -# Convenience function to clone only the necessary folders from a git repository -sparse_clone() { - repo_url=$1 - paths=$2 - commitish=$3 - clone_dir=$(basename "${repo_url%.*}") - rm -rf "${clone_dir}" - git clone -n --depth=1 --no-single-branch --filter=tree:0 "${repo_url}" - pushd "${clone_dir}" - if [ -n "${commitish}" ]; then - git checkout "${commitish}" - fi - git sparse-checkout set --no-cone ${paths} - git checkout - popd -} - -# calls a function in utilities.py. THe first argument is the function name, the -# rest of the arguments are the positional arguments to such function -py_util() { - python3 "${utilities_script_dir}/utilities.py" "$@" -} - -download_googleapis_files_and_folders() { - local output_folder=$1 - local googleapis_commitish=$2 - # checkout the master branch of googleapis/google (proto files) and WORKSPACE - echo "Checking out googlapis repository..." - # sparse_clone will remove folder contents first, so we have to checkout googleapis - # only once. - sparse_clone https://github.com/googleapis/googleapis.git "google grafeas" "${googleapis_commitish}" - pushd googleapis - cp -r google "${output_folder}" - cp -r grafeas "${output_folder}" -} - -get_gapic_generator_location() { - echo "${GAPIC_GENERATOR_LOCATION}" -} - -get_java_formatter_location() { - echo "${JAVA_FORMATTER_LOCATION}" -} - -error_if_not_exists() { - local required_tool=$1 - if [[ ! -f "${required_tool}" ]]; then - >&2 echo "File ${required_tool} not found in the filesystem. " - >&2 echo "Please configure your environment and store the " - >&2 echo "required tools in this location." - exit 1 - fi -} From 1e2473b06461b3297948703e59b3cef8d551e57a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 13:19:10 -0400 Subject: [PATCH 17/69] add library_generation module --- .../library_generation/cli/entry_point.py | 244 +++++++ .../cli/generate_monorepo_gapic_bom.py | 49 ++ .../cli/generate_monorepo_root_pom.py | 39 ++ .../library_generation/dockerignore | 6 + .../gapic-generator-java-wrapper | 7 + .../generate_composed_library.py | 173 +++++ .../library_generation/generate_library.sh | 278 ++++++++ .../library_generation/generate_repo.py | 88 +++ .../library_generation/model/bom_config.py | 45 ++ .../library_generation/model/config_change.py | 203 ++++++ .../library_generation/model/repo_config.py | 80 +++ .../owlbot/bin/entrypoint.sh | 78 +++ .../owlbot/bin/format_source.sh | 47 ++ .../owlbot/bin/write_clirr_ignore.sh | 34 + .../owlbot/src/fix-license-headers.py | 30 + .../library_generation/owlbot/src/fix_poms.py | 611 ++++++++++++++++++ .../owlbot/src/gen-template.py | 84 +++ .../owlbot/src/poms/.gitignore | 1 + .../owlbot/src/poms/module.py | 54 ++ .../owlbot/src/poms/templates.py | 35 + .../owlbot/synthtool/__init__.py | 32 + .../owlbot/synthtool/_tracked_paths.py | 39 ++ .../owlbot/synthtool/gcp/common.py | 146 +++++ .../owlbot/synthtool/gcp/samples.py | 91 +++ .../owlbot/synthtool/gcp/snippets.py | 124 ++++ .../owlbot/synthtool/languages/java.py | 611 ++++++++++++++++++ .../owlbot/synthtool/sources/templates.py | 79 +++ .../owlbot/synthtool/transforms.py | 314 +++++++++ .../clirr/clirr-ignored-differences.xml.j2 | 80 +++ .../templates/java_library/.github/CODEOWNERS | 20 + .../.github/ISSUE_TEMPLATE/bug_report.md | 56 ++ .../.github/ISSUE_TEMPLATE/feature_request.md | 26 + .../.github/ISSUE_TEMPLATE/support_request.md | 7 + .../.github/PULL_REQUEST_TEMPLATE.md | 10 + .../java_library/.github/auto-label.yaml | 15 + .../java_library/.github/blunderbuss.yml | 7 + .../java_library/.github/dependabot.yml | 19 + .../java_library/.github/release-please.yml | 3 + .../java_library/.github/release-trigger.yml | 2 + .../scripts/update_generation_config.sh | 121 ++++ .../java_library/.github/snippet-bot.yml | 0 .../.github/sync-repo-settings.yaml | 64 ++ .../.github/trusted-contribution.yml | 9 + .../.github/workflows/approve-readme.yaml | 69 ++ .../java_library/.github/workflows/ci.yaml | 123 ++++ .../workflows/renovate_config_check.yaml | 25 + .../.github/workflows/samples.yaml | 30 + .../workflows/update_generation_config.yaml | 43 ++ .../templates/java_library/.kokoro/build.sh | 28 + .../templates/java_library/.kokoro/common.cfg | 19 + .../java_library/.kokoro/continuous.cfg | 1 + .../java_library/.kokoro/presubmit.cfg | 1 + .../java_library/.kokoro/trampoline.sh | 24 + .../templates/java_library/CODE_OF_CONDUCT.md | 94 +++ .../templates/java_library/CONTRIBUTING.md | 92 +++ .../owlbot/templates/java_library/LICENSE | 201 ++++++ .../owlbot/templates/java_library/README.md | 288 +++++++++ .../owlbot/templates/java_library/SECURITY.md | 7 + .../owlbot/templates/java_library/java.header | 15 + .../templates/java_library/license-checks.xml | 10 + .../templates/java_library/renovate.json | 128 ++++ .../samples/install-without-bom/pom.xml | 86 +++ .../templates/java_library/samples/pom.xml | 56 ++ .../java_library/samples/snapshot/pom.xml | 85 +++ .../java_library/samples/snippets/pom.xml | 49 ++ .../owlbot/templates/poms/bom_pom.xml.j2 | 41 ++ .../owlbot/templates/poms/cloud_pom.xml.j2 | 156 +++++ .../owlbot/templates/poms/grpc_pom.xml.j2 | 71 ++ .../owlbot/templates/poms/parent_pom.xml.j2 | 51 ++ .../owlbot/templates/poms/proto_pom.xml.j2 | 48 ++ .../owlbot/templates/poms/versions.txt.j2 | 4 + .../library_generation/postprocess_library.sh | 111 ++++ hermetic_build/library_generation/setup.py | 42 ++ .../templates/gapic-libraries-bom.xml.j2 | 37 ++ .../library_generation/templates/owlbot.py.j2 | 28 + .../templates/owlbot.yaml.monorepo.j2 | 36 ++ .../templates/root-pom.xml.j2 | 88 +++ .../library_generation/tests/__init__.py | 0 .../library_generation/tests/cli/__init__.py | 0 .../tests/cli/entry_point_unit_tests.py | 416 ++++++++++++ .../library_generation/tests/compare_poms.py | 131 ++++ .../tests/generate_library_unit_tests.py | 109 ++++ .../tests/generate_library_unit_tests.sh | 314 +++++++++ .../tests/generate_repo_unit_tests.py | 73 +++ .../tests/integration_tests.py | 394 +++++++++++ .../tests/model/__init__.py | 0 .../tests/model/config_change_unit_tests.py | 314 +++++++++ .../tests/model/repo_config_unit_tests.py | 56 ++ .../tests/owlbot/__init__.py | 0 .../tests/owlbot/fix_poms_unit_tests.py | 47 ++ .../tests/owlbot/java_unit_tests.py | 310 +++++++++ .../library_generation/tests/owlbot/util.py | 126 ++++ .../gapic_options/QueryServiceGrpc_copy.java | 9 + .../resources/gapic_options/example.yaml | 1 + .../gapic_options/example_gapic.legacy.yaml | 1 + .../gapic_options/example_gapic.yaml | 1 + .../gapic_options/example_gapic_legacy.yaml | 1 + .../example_grpc_service_config.json | 3 + .../goldens/.OwlBot-hermetic-golden.yaml | 35 + ...repo-metadata-custom-transport-golden.json | 16 + .../.repo-metadata-monorepo-golden.json | 20 + .../.repo-metadata-non-monorepo-golden.json | 21 + .../.repo-metadata-proto-only-golden.json | 19 + .../tests/resources/goldens/owlbot-golden.py | 36 ++ .../goldens/pr_description-golden.txt | 17 + ...qualified_commit_pr_description-golden.txt | 10 + .../repo_level_only_pr_description-golden.txt | 5 + .../baseline_generation_config.yaml | 45 ++ .../current_generation_config.yaml | 45 ++ .../pr-description-golden.txt | 54 ++ .../java-bigtable/generation_config.yaml | 22 + .../java-bigtable/pr-description-golden.txt | 18 + .../integration/test_generator_coordinates | 1 + .../misc/BUILD_comment_common_resources.bazel | 5 + .../misc/BUILD_comment_iam_policy.bazel | 5 + .../misc/BUILD_comment_locations.bazel | 5 + .../misc/BUILD_common_resources.bazel | 5 + .../resources/misc/BUILD_gapic_yaml.bazel | 3 + .../tests/resources/misc/BUILD_grpc.bazel | 5 + .../resources/misc/BUILD_grpc_rest.bazel | 5 + .../resources/misc/BUILD_iam_locations.bazel | 6 + .../resources/misc/BUILD_iam_policy.bazel | 5 + .../misc/BUILD_include_samples_empty.bazel | 5 + .../misc/BUILD_include_samples_false.bazel | 5 + .../misc/BUILD_include_samples_true.bazel | 5 + .../resources/misc/BUILD_locations.bazel | 5 + .../misc/BUILD_no_additional_protos.bazel | 4 + .../resources/misc/BUILD_no_gapic_yaml.bazel | 3 + .../misc/BUILD_no_service_config.bazel | 3 + .../misc/BUILD_no_service_yaml.bazel | 3 + .../resources/misc/BUILD_proto_only.bazel | 16 + .../tests/resources/misc/BUILD_rest.bazel | 5 + .../misc/BUILD_rest_numeric_enums_empty.bazel | 5 + .../misc/BUILD_rest_numeric_enums_false.bazel | 5 + .../misc/BUILD_rest_numeric_enums_true.bazel | 5 + .../resources/misc/BUILD_service_config.bazel | 3 + ...BUILD_service_config_relative_target.bazel | 3 + .../resources/misc/BUILD_service_yaml.bazel | 3 + .../BUILD_service_yaml_absolute_target.bazel | 3 + .../tests/resources/misc/TESTWORKSPACE | 133 ++++ .../tests/resources/misc/versions.txt | 9 + .../proto/google/cloud/test/v1/empty.proto | 0 .../proto-1/fake.proto | 0 .../proto-2/fake.proto | 0 .../config_without_api_description.yaml | 5 + .../config_without_api_shortname.yaml | 4 + .../config_without_gapics_key.yaml | 3 + .../config_without_gapics_value.yaml | 4 + .../config_without_googleapis.yaml | 8 + .../test-config/config_without_libraries.yaml | 1 + .../config_without_library_value.yaml | 2 + .../config_without_name_pretty.yaml | 6 + .../config_without_product_docs.yaml | 7 + .../config_without_proto_path.yaml | 4 + .../config_without_temp_excludes.yaml | 10 + .../test-config/generation_config.yaml | 24 + .../generation_config_library_modified.yaml | 14 + ...on_config_with_duplicate_library_name.yaml | 51 ++ .../test-config/monorepo_baseline.yaml | 30 + .../test-config/monorepo_current.yaml | 30 + .../monorepo_with_common_protos.yaml | 43 ++ .../monorepo_without_common_protos.yaml | 33 + .../test-monorepo/.github/.OwlBot.lock.yaml | 17 + .../test-service/.repo-metadata.json | 18 + .../.github/release-please.yml | 6 + .../release-please-update/.repo-metadata.json | 18 + .../render-readme/.repo-metadata.json | 18 + .../standard/.repo-metadata.json | 18 + .../java-admanager/.repo-metadata.json | 16 + .../ad-manager-bom/pom-golden.xml | 38 ++ .../java-admanager/ad-manager/pom-golden.xml | 110 ++++ .../test-owlbot/java-admanager/pom-golden.xml | 49 ++ .../proto-ad-manager-v1/pom-golden.xml | 37 ++ .../test-owlbot/java-admanager/versions.txt | 6 + .../test-owlbot/testdata/FooGrpcGolden.java | 22 + .../test-owlbot/testdata/FooProtoGolden.java | 25 + .../test-owlbot/testdata/README-golden.md | 202 ++++++ .../test-owlbot/testdata/SampleClass.java | 29 + .../testdata/SampleClassGolden.java | 23 + .../testdata/SampleCopyMethodGolden.java | 37 ++ .../testdata/SampleDeprecateClass.java | 40 ++ .../testdata/SampleDeprecateMethodGolden.java | 52 ++ .../test-owlbot/testdata/src/foo/FooGrpc.java | 7 + .../testdata/src/foo/FooProto.java | 10 + .../empty_gen_config.yaml | 0 .../gapic-libraries-bom/pom-golden.xml | 45 ++ .../java-dns/pom.xml | 9 + .../google-cloud-service-control-bom/pom.xml | 8 + .../java-tasks/google-cloud-tasks-bom/pom.xml | 8 + .../pom-golden.xml | 88 +++ .../test_monorepo_postprocessing/versions.txt | 4 + .../src/main/java/example_main.txt | 0 .../src/test/java/example_test.txt | 0 .../src/main/java/example_proto_main.txt | 0 .../src/main/java/com/example_com_sample.txt | 0 .../src/main/java/io/example_io_sample.txt | 0 .../tests/test_utilities.sh | 129 ++++ .../library_generation/tests/test_utils.py | 40 ++ .../tests/utilities_unit_tests.py | 391 +++++++++++ .../tests/utils/__init__.py | 0 ...generation_config_comparator_unit_tests.py | 506 +++++++++++++++ .../monorepo_postprocessor_unit_tests.py | 46 ++ .../tests/utils/pom_generator_unit_tests.py | 32 + .../utils/proto_path_utils_unit_tests.py | 50 ++ .../library_generation/utils/file_render.py | 24 + .../utils/generation_config_comparator.py | 284 ++++++++ .../utils/monorepo_postprocessor.py | 29 + .../library_generation/utils/pom_generator.py | 157 +++++ .../utils/proto_path_utils.py | 47 ++ .../library_generation/utils/utilities.py | 309 +++++++++ .../library_generation/utils/utilities.sh | 391 +++++++++++ library_generation/DEVELOPMENT.md | 207 ------ library_generation/README.md | 310 --------- 213 files changed, 12859 insertions(+), 517 deletions(-) create mode 100644 hermetic_build/library_generation/cli/entry_point.py create mode 100644 hermetic_build/library_generation/cli/generate_monorepo_gapic_bom.py create mode 100644 hermetic_build/library_generation/cli/generate_monorepo_root_pom.py create mode 100644 hermetic_build/library_generation/dockerignore create mode 100755 hermetic_build/library_generation/gapic-generator-java-wrapper create mode 100755 hermetic_build/library_generation/generate_composed_library.py create mode 100755 hermetic_build/library_generation/generate_library.sh create mode 100755 hermetic_build/library_generation/generate_repo.py create mode 100644 hermetic_build/library_generation/model/bom_config.py create mode 100644 hermetic_build/library_generation/model/config_change.py create mode 100644 hermetic_build/library_generation/model/repo_config.py create mode 100755 hermetic_build/library_generation/owlbot/bin/entrypoint.sh create mode 100755 hermetic_build/library_generation/owlbot/bin/format_source.sh create mode 100755 hermetic_build/library_generation/owlbot/bin/write_clirr_ignore.sh create mode 100644 hermetic_build/library_generation/owlbot/src/fix-license-headers.py create mode 100644 hermetic_build/library_generation/owlbot/src/fix_poms.py create mode 100644 hermetic_build/library_generation/owlbot/src/gen-template.py create mode 100644 hermetic_build/library_generation/owlbot/src/poms/.gitignore create mode 100644 hermetic_build/library_generation/owlbot/src/poms/module.py create mode 100644 hermetic_build/library_generation/owlbot/src/poms/templates.py create mode 100644 hermetic_build/library_generation/owlbot/synthtool/__init__.py create mode 100644 hermetic_build/library_generation/owlbot/synthtool/_tracked_paths.py create mode 100644 hermetic_build/library_generation/owlbot/synthtool/gcp/common.py create mode 100644 hermetic_build/library_generation/owlbot/synthtool/gcp/samples.py create mode 100644 hermetic_build/library_generation/owlbot/synthtool/gcp/snippets.py create mode 100644 hermetic_build/library_generation/owlbot/synthtool/languages/java.py create mode 100644 hermetic_build/library_generation/owlbot/synthtool/sources/templates.py create mode 100644 hermetic_build/library_generation/owlbot/synthtool/transforms.py create mode 100644 hermetic_build/library_generation/owlbot/templates/clirr/clirr-ignored-differences.xml.j2 create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/CODEOWNERS create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/feature_request.md create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/support_request.md create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/PULL_REQUEST_TEMPLATE.md create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/auto-label.yaml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/blunderbuss.yml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/dependabot.yml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/release-please.yml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/release-trigger.yml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/scripts/update_generation_config.sh create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/snippet-bot.yml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/trusted-contribution.yml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/approve-readme.yaml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/ci.yaml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/samples.yaml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml create mode 100755 hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/build.sh create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/common.cfg create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/continuous.cfg create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit.cfg create mode 100755 hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/trampoline.sh create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/CONTRIBUTING.md create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/LICENSE create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/README.md create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/SECURITY.md create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/java.header create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/license-checks.xml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/renovate.json create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/samples/pom.xml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml create mode 100644 hermetic_build/library_generation/owlbot/templates/java_library/samples/snippets/pom.xml create mode 100644 hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 create mode 100644 hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 create mode 100644 hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 create mode 100644 hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 create mode 100644 hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 create mode 100644 hermetic_build/library_generation/owlbot/templates/poms/versions.txt.j2 create mode 100755 hermetic_build/library_generation/postprocess_library.sh create mode 100755 hermetic_build/library_generation/setup.py create mode 100644 hermetic_build/library_generation/templates/gapic-libraries-bom.xml.j2 create mode 100644 hermetic_build/library_generation/templates/owlbot.py.j2 create mode 100644 hermetic_build/library_generation/templates/owlbot.yaml.monorepo.j2 create mode 100644 hermetic_build/library_generation/templates/root-pom.xml.j2 create mode 100644 hermetic_build/library_generation/tests/__init__.py create mode 100644 hermetic_build/library_generation/tests/cli/__init__.py create mode 100644 hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py create mode 100644 hermetic_build/library_generation/tests/compare_poms.py create mode 100644 hermetic_build/library_generation/tests/generate_library_unit_tests.py create mode 100755 hermetic_build/library_generation/tests/generate_library_unit_tests.sh create mode 100644 hermetic_build/library_generation/tests/generate_repo_unit_tests.py create mode 100644 hermetic_build/library_generation/tests/integration_tests.py create mode 100644 hermetic_build/library_generation/tests/model/__init__.py create mode 100644 hermetic_build/library_generation/tests/model/config_change_unit_tests.py create mode 100644 hermetic_build/library_generation/tests/model/repo_config_unit_tests.py create mode 100644 hermetic_build/library_generation/tests/owlbot/__init__.py create mode 100644 hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py create mode 100644 hermetic_build/library_generation/tests/owlbot/java_unit_tests.py create mode 100644 hermetic_build/library_generation/tests/owlbot/util.py create mode 100644 hermetic_build/library_generation/tests/resources/gapic_options/QueryServiceGrpc_copy.java create mode 100644 hermetic_build/library_generation/tests/resources/gapic_options/example.yaml create mode 100644 hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.legacy.yaml create mode 100644 hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.yaml create mode 100644 hermetic_build/library_generation/tests/resources/gapic_options/example_gapic_legacy.yaml create mode 100644 hermetic_build/library_generation/tests/resources/gapic_options/example_grpc_service_config.json create mode 100644 hermetic_build/library_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml create mode 100644 hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json create mode 100644 hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json create mode 100644 hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json create mode 100644 hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json create mode 100644 hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py create mode 100644 hermetic_build/library_generation/tests/resources/goldens/pr_description-golden.txt create mode 100644 hermetic_build/library_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt create mode 100644 hermetic_build/library_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt create mode 100644 hermetic_build/library_generation/tests/resources/integration/google-cloud-java/baseline_generation_config.yaml create mode 100644 hermetic_build/library_generation/tests/resources/integration/google-cloud-java/current_generation_config.yaml create mode 100644 hermetic_build/library_generation/tests/resources/integration/google-cloud-java/pr-description-golden.txt create mode 100644 hermetic_build/library_generation/tests/resources/integration/java-bigtable/generation_config.yaml create mode 100644 hermetic_build/library_generation/tests/resources/integration/java-bigtable/pr-description-golden.txt create mode 100644 hermetic_build/library_generation/tests/resources/integration/test_generator_coordinates create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_comment_common_resources.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_comment_iam_policy.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_comment_locations.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_common_resources.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_gapic_yaml.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_grpc.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_grpc_rest.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_iam_locations.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_iam_policy.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_empty.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_false.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_true.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_locations.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_no_additional_protos.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_no_gapic_yaml.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_config.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_yaml.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_proto_only.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_rest.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_service_config.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_service_config_relative_target.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel create mode 100644 hermetic_build/library_generation/tests/resources/misc/TESTWORKSPACE create mode 100644 hermetic_build/library_generation/tests/resources/misc/versions.txt create mode 100644 hermetic_build/library_generation/tests/resources/proto_path_library/proto-test-library/src/main/proto/google/cloud/test/v1/empty.proto create mode 100644 hermetic_build/library_generation/tests/resources/proto_path_library_multiple_protos/proto-1/fake.proto create mode 100644 hermetic_build/library_generation/tests/resources/proto_path_library_multiple_protos/proto-2/fake.proto create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_api_description.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_api_shortname.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_key.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_value.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_googleapis.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_libraries.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_library_value.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_name_pretty.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_product_docs.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_proto_path.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_temp_excludes.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/generation_config.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/generation_config_library_modified.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/monorepo_baseline.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/monorepo_current.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/monorepo_with_common_protos.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-config/monorepo_without_common_protos.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-monorepo/.github/.OwlBot.lock.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test-monorepo/test-service/.repo-metadata.json create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/release-please-update/.github/release-please.yml create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/release-please-update/.repo-metadata.json create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/render-readme/.repo-metadata.json create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/standard/.repo-metadata.json create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/.repo-metadata.json create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/ad-manager-bom/pom-golden.xml create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/pom-golden.xml create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/proto-ad-manager-v1/pom-golden.xml create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/testdata/FooGrpcGolden.java create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/testdata/FooProtoGolden.java create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/testdata/README-golden.md create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleClass.java create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleClassGolden.java create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleCopyMethodGolden.java create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleDeprecateClass.java create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleDeprecateMethodGolden.java create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/testdata/src/foo/FooGrpc.java create mode 100644 hermetic_build/library_generation/tests/resources/test-owlbot/testdata/src/foo/FooProto.java create mode 100644 hermetic_build/library_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml create mode 100644 hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml create mode 100644 hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-dns/pom.xml create mode 100644 hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml create mode 100644 hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml create mode 100644 hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/pom-golden.xml create mode 100644 hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/versions.txt create mode 100644 hermetic_build/library_generation/tests/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt create mode 100644 hermetic_build/library_generation/tests/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt create mode 100644 hermetic_build/library_generation/tests/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt create mode 100644 hermetic_build/library_generation/tests/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt create mode 100644 hermetic_build/library_generation/tests/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt create mode 100755 hermetic_build/library_generation/tests/test_utilities.sh create mode 100644 hermetic_build/library_generation/tests/test_utils.py create mode 100644 hermetic_build/library_generation/tests/utilities_unit_tests.py create mode 100644 hermetic_build/library_generation/tests/utils/__init__.py create mode 100644 hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py create mode 100644 hermetic_build/library_generation/tests/utils/monorepo_postprocessor_unit_tests.py create mode 100644 hermetic_build/library_generation/tests/utils/pom_generator_unit_tests.py create mode 100644 hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py create mode 100644 hermetic_build/library_generation/utils/file_render.py create mode 100644 hermetic_build/library_generation/utils/generation_config_comparator.py create mode 100644 hermetic_build/library_generation/utils/monorepo_postprocessor.py create mode 100644 hermetic_build/library_generation/utils/pom_generator.py create mode 100644 hermetic_build/library_generation/utils/proto_path_utils.py create mode 100755 hermetic_build/library_generation/utils/utilities.py create mode 100755 hermetic_build/library_generation/utils/utilities.sh delete mode 100644 library_generation/DEVELOPMENT.md delete mode 100644 library_generation/README.md diff --git a/hermetic_build/library_generation/cli/entry_point.py b/hermetic_build/library_generation/cli/entry_point.py new file mode 100644 index 0000000000..b15880f06d --- /dev/null +++ b/hermetic_build/library_generation/cli/entry_point.py @@ -0,0 +1,244 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +from typing import Optional +import click as click +from library_generation.generate_repo import generate_from_yaml +from library_generation.model.config_change import ConfigChange +from common.model.generation_config import from_yaml +from library_generation.utils.generation_config_comparator import compare_config + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--baseline-generation-config-path", + required=False, + default=None, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml. + This config file is used for commit history generation, not library + generation. + """, +) +@click.option( + "--current-generation-config-path", + required=False, + default=None, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml that contains the + metadata about library generation. + """, +) +@click.option( + "--library-names", + type=str, + default=None, + show_default=True, + help=""" + A list of library names that will be generated, separated by comma. + The library name of a library is the value of library_name or api_shortname, + if library_name is not specified, in the generation configuration. + """, +) +@click.option( + "--repository-path", + type=str, + default=".", + show_default=True, + help=""" + The repository path to which the generated files + will be sent. + If not specified, the repository will be generated to the current working + directory. + """, +) +@click.option( + "--api-definitions-path", + type=str, + default=".", + show_default=True, + help=""" + The path to which the api definition (proto and service yaml) and its + dependencies resides. + If not specified, the path is the current working directory. + """, +) +def generate( + baseline_generation_config_path: str, + current_generation_config_path: str, + library_names: Optional[str], + repository_path: str, + api_definitions_path: str, +): + """ + Compare baseline generation config and current generation config and + generate changed libraries based on current generation config. + + If baseline generation config is not specified but current generation + config is specified, generate all libraries if `library_names` is not + specified, based on current generation config. + + If current generation config is not specified but baseline generation + config is specified, raise FileNotFoundError because current generation + config should be the source of truth of library generation. + + If both baseline generation config and current generation config are not + specified, generate all libraries based on the default generation config, + which is generation_config.yaml in the current working directory. + + If `library_names` is specified, only libraries whose name can be found in + the current generation config or default generation config, if current + generation config is not specified, will be generated. Changed libraries + will be ignored even if baseline and current generation config are + specified. + + Raise FileNotFoundError if the default config does not exist. + """ + __generate_repo_and_pr_description_impl( + baseline_generation_config_path=baseline_generation_config_path, + current_generation_config_path=current_generation_config_path, + library_names=library_names, + repository_path=repository_path, + api_definitions_path=api_definitions_path, + ) + + +def __generate_repo_and_pr_description_impl( + baseline_generation_config_path: str, + current_generation_config_path: str, + library_names: Optional[str], + repository_path: str, + api_definitions_path: str, +): + """ + Implementation method for generate(). + The decoupling of generate and __generate_repo_and_pr_description_impl is + meant to allow testing of this implementation function. + """ + + default_generation_config_path = f"{os.getcwd()}/generation_config.yaml" + + if ( + baseline_generation_config_path is None + and current_generation_config_path is None + ): + if not os.path.isfile(default_generation_config_path): + raise FileNotFoundError( + f"{default_generation_config_path} does not exist. " + "A valid generation config has to be passed in as " + "current_generation_config or exist in the current working " + "directory." + ) + current_generation_config_path = default_generation_config_path + elif current_generation_config_path is None: + raise FileNotFoundError( + "current_generation_config is not specified when " + "baseline_generation_config is specified. " + "current_generation_config should be the source of truth of " + "library generation." + ) + + current_generation_config_path = os.path.abspath(current_generation_config_path) + repository_path = os.path.abspath(repository_path) + api_definitions_path = os.path.abspath(api_definitions_path) + include_library_names = _parse_library_name_from(library_names) + + if not baseline_generation_config_path: + # Execute selective generation based on current_generation_config if + # baseline_generation_config is not specified. + generate_from_yaml( + config=from_yaml(current_generation_config_path), + repository_path=repository_path, + api_definitions_path=api_definitions_path, + target_library_names=include_library_names, + ) + return + + # Compare two generation configs to get changed libraries. + baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) + config_change = compare_config( + baseline_config=from_yaml(baseline_generation_config_path), + current_config=from_yaml(current_generation_config_path), + ) + # Pass None if we want to fully generate the repository. + changed_library_names = ( + config_change.get_changed_libraries() + if not _needs_full_repo_generation(config_change=config_change) + else None + ) + # Include library names takes preference if specified. + target_library_names = ( + include_library_names + if include_library_names is not None + else changed_library_names + ) + generate_from_yaml( + config=config_change.current_config, + repository_path=repository_path, + api_definitions_path=api_definitions_path, + target_library_names=target_library_names, + ) + + +def _needs_full_repo_generation(config_change: ConfigChange) -> bool: + """ + Whether you should need a full repo generation, i.e., generate all + libraries in the generation configuration. + """ + current_config = config_change.current_config + return not current_config.is_monorepo() or current_config.contains_common_protos() + + +def _parse_library_name_from(includes: str) -> Optional[list[str]]: + if includes is None: + return None + return [library_name.strip() for library_name in includes.split(",")] + + +@main.command() +@click.option( + "--generation-config-path", + required=False, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml. + Default to generation_config.yaml in the current working directory. + """, +) +def validate_generation_config(generation_config_path: str) -> None: + """ + Validate the given generation configuration. + """ + if generation_config_path is None: + generation_config_path = "generation_config.yaml" + try: + from_yaml(os.path.abspath(generation_config_path)) + print(f"{generation_config_path} is validated without any errors.") + except ValueError as err: + print(err) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/hermetic_build/library_generation/cli/generate_monorepo_gapic_bom.py b/hermetic_build/library_generation/cli/generate_monorepo_gapic_bom.py new file mode 100644 index 0000000000..b26a02c17f --- /dev/null +++ b/hermetic_build/library_generation/cli/generate_monorepo_gapic_bom.py @@ -0,0 +1,49 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import click as click + +from library_generation.utils.pom_generator import generate_gapic_bom + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--repository-path", + required=True, + type=str, + help=""" + Path to which the generated pom.xml goes. + """, +) +@click.option( + "--versions-file", + required=True, + type=str, + help=""" + The file containing version of libraries. + Throw FileNotFoundError if the file doesn't exist. + """, +) +def generate(repository_path: str, versions_file: str) -> None: + generate_gapic_bom(repository_path=repository_path, versions_file=versions_file) + + +if __name__ == "__main__": + main() diff --git a/hermetic_build/library_generation/cli/generate_monorepo_root_pom.py b/hermetic_build/library_generation/cli/generate_monorepo_root_pom.py new file mode 100644 index 0000000000..8f129b63f0 --- /dev/null +++ b/hermetic_build/library_generation/cli/generate_monorepo_root_pom.py @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import click as click +from library_generation.utils.pom_generator import generate_root_pom + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--repository-path", + required=True, + type=str, + help=""" + Path to which the generated pom.xml goes. + """, +) +def generate(repository_path: str) -> None: + generate_root_pom(repository_path=repository_path) + + +if __name__ == "__main__": + main() diff --git a/hermetic_build/library_generation/dockerignore b/hermetic_build/library_generation/dockerignore new file mode 100644 index 0000000000..7e978caea9 --- /dev/null +++ b/hermetic_build/library_generation/dockerignore @@ -0,0 +1,6 @@ +README.md +**/__pycache__/ +**/*.egg-info/ +**/output/ +**/build/ +**/google-cloud-java/ diff --git a/hermetic_build/library_generation/gapic-generator-java-wrapper b/hermetic_build/library_generation/gapic-generator-java-wrapper new file mode 100755 index 0000000000..3f1e7bd2e7 --- /dev/null +++ b/hermetic_build/library_generation/gapic-generator-java-wrapper @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -e +wrapper_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +source "${wrapper_dir}/utils/utilities.sh" + +# Wrap gapic-generator-java.jar because protoc requires the plugin to be executable. +exec java -classpath "$(get_gapic_generator_location)" com.google.api.generator.Main diff --git a/hermetic_build/library_generation/generate_composed_library.py b/hermetic_build/library_generation/generate_composed_library.py new file mode 100755 index 0000000000..7cdc9c53d7 --- /dev/null +++ b/hermetic_build/library_generation/generate_composed_library.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script allows generation of libraries that are composed of more than one +service version. It is achieved by calling `generate_library.sh` without +postprocessing for all service versions and then calling +postprocess_library.sh at the end, once all libraries are ready. + +Prerequisites +- Needs a folder named `output` in current working directory. This folder +is automatically detected by `generate_library.sh` and this script ensures it +contains the necessary folders and files, specifically: + - A "google" folder found in the googleapis/googleapis repository + - A "grafeas" folder found in the googleapis/googleapis repository +Note: googleapis repo is found in https://github.com/googleapis/googleapis. +""" +import os +from pathlib import Path +from typing import List +import library_generation.utils.utilities as util +from common.model.generation_config import GenerationConfig +from common.model.gapic_config import GapicConfig +from common.model.gapic_inputs import GapicInputs +from common.model.library_config import LibraryConfig +from common.model.gapic_inputs import parse as parse_build_file +from library_generation.model.repo_config import RepoConfig + +script_dir = os.path.dirname(os.path.realpath(__file__)) + + +def generate_composed_library( + config: GenerationConfig, + library_path: str, + library: LibraryConfig, + repo_config: RepoConfig, +) -> None: + """ + Generate libraries composed of more than one service or service version + + :param config: a GenerationConfig object representing a parsed configuration + yaml + :param library_path: the path to which the generated file goes + :param library: a LibraryConfig object contained inside config, passed here + for convenience and to prevent all libraries to be processed + :param repo_config: + :return None + """ + output_folder = repo_config.output_folder + base_arguments = __construct_tooling_arg(config=config) + owlbot_cli_source_folder = util.sh_util("mktemp -d") + os.makedirs(f"{library_path}", exist_ok=True) + for gapic in library.get_sorted_gapic_configs(): + build_file_folder = Path(f"{output_folder}/{gapic.proto_path}").resolve() + print(f"build_file_folder: {build_file_folder}") + gapic_inputs = parse_build_file(build_file_folder, gapic.proto_path) + # generate postprocessing prerequisite files (.repo-metadata.json, .OwlBot-hermetic.yaml, + # owlbot.py) here because transport is parsed from BUILD.bazel, + # which lives in a versioned proto_path. The value of transport will be + # overridden by the config object if specified. Note that this override + # does not affect library generation but instead used only for + # generating postprocessing files such as README. + util.generate_postprocessing_prerequisite_files( + config=config, + library=library, + proto_path=util.remove_version_from(gapic.proto_path), + library_path=library_path, + transport=library.get_transport(gapic_inputs), + ) + temp_destination_path = f"java-{gapic.proto_path.replace('/','-')}" + effective_arguments = __construct_effective_arg( + base_arguments=base_arguments, + gapic=gapic, + gapic_inputs=gapic_inputs, + temp_destination_path=temp_destination_path, + ) + print("arguments: ") + print(effective_arguments) + print(f"Generating library from {gapic.proto_path} to {library_path}") + util.run_process_and_print_output( + ["bash", f"{script_dir}/generate_library.sh", *effective_arguments], + "Library generation", + ) + + util.sh_util( + f'build_owlbot_cli_source_folder "{library_path}"' + + f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"' + + f' "{gapic.proto_path}"', + cwd=output_folder, + ) + + library_version = repo_config.get_library_version( + artifact_id=library.get_artifact_id() + ) + # call postprocess library + util.run_process_and_print_output( + [ + f"{script_dir}/postprocess_library.sh", + f"{library_path}", + "", + repo_config.versions_file, + owlbot_cli_source_folder, + str(config.is_monorepo()).lower(), + config.libraries_bom_version, + library_version, + ], + "Library postprocessing", + ) + + +def __construct_tooling_arg(config: GenerationConfig) -> List[str]: + """ + Construct arguments of tooling versions used in generate_library.sh + :param config: the generation config + :return: arguments containing tooling versions + """ + arguments = [] + arguments += util.create_argument("grpc_version", config) + arguments += util.create_argument("protoc_version", config) + + return arguments + + +def __construct_effective_arg( + base_arguments: List[str], + gapic: GapicConfig, + gapic_inputs: GapicInputs, + temp_destination_path: str, +) -> List[str]: + """ + Construct arguments consist attributes of a GAPIC library which used in + generate_library.sh + :param base_arguments: arguments consist of tooling versions + :param gapic: an object of GapicConfig + :param gapic_inputs: an object of GapicInput + :param temp_destination_path: the path to which the generated library goes + :return: arguments containing attributes to generate a GAPIC library + """ + arguments = list(base_arguments) + arguments += util.create_argument("proto_path", gapic) + arguments += [ + "--proto_only", + gapic_inputs.proto_only, + "--gapic_additional_protos", + gapic_inputs.additional_protos, + "--transport", + gapic_inputs.transport, + "--rest_numeric_enums", + gapic_inputs.rest_numeric_enum, + "--gapic_yaml", + gapic_inputs.gapic_yaml, + "--service_config", + gapic_inputs.service_config, + "--service_yaml", + gapic_inputs.service_yaml, + "--include_samples", + gapic_inputs.include_samples, + ] + arguments += ["--destination_path", temp_destination_path] + + return arguments diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh new file mode 100755 index 0000000000..b8d22aca54 --- /dev/null +++ b/hermetic_build/library_generation/generate_library.sh @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# parse input parameters +while [[ $# -gt 0 ]]; do +key="$1" +case $key in + -p|--proto_path) + proto_path="$2" + shift + ;; + -d|--destination_path) + destination_path="$2" + shift + ;; + --protoc_version) + protoc_version="$2" + shift + ;; + --grpc_version) + grpc_version="$2" + shift + ;; + --proto_only) + proto_only="$2" + shift + ;; + --gapic_additional_protos) + gapic_additional_protos="$2" + shift + ;; + --transport) + transport="$2" + shift + ;; + --rest_numeric_enums) + rest_numeric_enums="$2" + shift + ;; + --gapic_yaml) + gapic_yaml="$2" + shift + ;; + --service_config) + service_config="$2" + shift + ;; + --service_yaml) + service_yaml="$2" + shift + ;; + --include_samples) + include_samples="$2" + shift + ;; + --os_architecture) + os_architecture="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift # past argument or value +done + +script_dir=$(dirname "$(readlink -f "$0")") +# source utility functions +source "${script_dir}"/utils/utilities.sh +output_folder="$(get_output_folder)" + +if [ -z "${protoc_version}" ]; then + protoc_version=$(get_protoc_version) +fi + +if [ -z "${grpc_version}" ]; then + grpc_version=$(get_grpc_version) +fi + +if [ -z "${proto_only}" ]; then + proto_only="false" +fi + +if [ -z "${gapic_additional_protos}" ]; then + gapic_additional_protos="google/cloud/common_resources.proto" +fi + +if [ -z "${transport}" ]; then + transport="grpc" +fi + +if [ -z "${rest_numeric_enums}" ]; then + rest_numeric_enums="true" +fi + +if [ -z "${gapic_yaml}" ]; then + gapic_yaml="" +fi + +if [ -z "${service_config}" ]; then + service_config="" +fi + +if [ -z "${service_yaml}" ]; then + service_yaml="" +fi + +if [ -z "${include_samples}" ]; then + include_samples="true" +fi + +if [ -z "${os_architecture}" ]; then + os_architecture=$(detect_os_architecture) +fi + +temp_destination_path="${output_folder}/temp_preprocessed" +mkdir -p "${output_folder}/${destination_path}" +if [ -d "${temp_destination_path}" ]; then + # we don't want the preprocessed sources of a previous run + rm -rd "${temp_destination_path}" +fi +mkdir -p "${temp_destination_path}" +##################### Section 0 ##################### +# prepare tooling +##################################################### +# the order of services entries in gapic_metadata.json is relevant to the +# order of proto file, sort the proto files with respect to their bytes to +# get a fixed order. +folder_name=$(extract_folder_name "${destination_path}") +pushd "${output_folder}" +find_depth="" +case "${proto_path}" in + "google/api" | "google/cloud" | "google/rpc") + find_depth="-maxdepth 1" + ;; +esac +proto_files=$(find "${proto_path}" ${find_depth} -type f -name "*.proto" | LC_COLLATE=C sort) +# include or exclude certain protos in grpc plugin and gapic generator java. +case "${proto_path}" in + "google/cloud") + # this proto is excluded from //google/cloud:google-apps-script-type-java + removed_proto="google/cloud/common_resources.proto" + proto_files="${proto_files//${removed_proto}/}" + ;; + "google/cloud/aiplatform/v1beta1"*) + # this proto is excluded from //google/cloud/aiplatform/v1beta1/schema:schema_proto + removed_proto="google/cloud/aiplatform/v1beta1/schema/io_format.proto" + proto_files="${proto_files//${removed_proto}/}" + ;; + "google/cloud/filestore"*) + # this proto is included in //google/cloud/filestore/v1:google-cloud-filestore-v1-java + # and //google/cloud/filestore/v1beta1:google-cloud-filestore-v1-java + proto_files="${proto_files} google/cloud/common/operation_metadata.proto" + ;; + "google/cloud/oslogin"*) + # this proto is included in //google/cloud/oslogin/v1:google-cloud-oslogin-v1-java + # and //google/cloud/oslogin/v1beta1:google-cloud-oslogin-v1-java + proto_files="${proto_files} google/cloud/oslogin/common/common.proto" + ;; + "google/cloud/visionai/v1"*) + # this proto is excluded in //google/cloud/visionai/v1:google-cloud-visionai-v1-java + # we can remove this exclusion after cl/631529749 is submitted. + removed_proto="google/cloud/visionai/v1/prediction.proto" + proto_files="${proto_files//${removed_proto}/}" + ;; + "google/rpc") + # this proto is excluded from //google/rpc:google-rpc-java + removed_proto="google/rpc/http.proto" + proto_files="${proto_files//${removed_proto}/}" + ;; +esac +# download gapic-generator-java, protobuf and grpc plugin. +# the download_tools function will create the environment variables "protoc_path" +# and "grpc_path", to be used in the protoc calls below. +download_tools "${protoc_version}" "${grpc_version}" "${os_architecture}" +##################### Section 1 ##################### +# generate grpc-*/ +##################################################### +if [[ ! "${transport}" == "rest" ]]; then + # do not need to generate grpc-* if the transport is `rest`. + "${protoc_path}"/protoc "--plugin=protoc-gen-rpc-plugin=${grpc_path}" \ + "--rpc-plugin_out=:${temp_destination_path}/java_grpc.jar" \ + ${proto_files} # Do not quote because this variable should not be treated as one long string. + # unzip java_grpc.jar to grpc-*/src/main/java + unzip_src_files "grpc" "${temp_destination_path}" + # remove empty files in grpc-*/src/main/java + remove_empty_files "grpc" "${temp_destination_path}" + # remove grpc version in *ServiceGrpc.java file so the content is identical with bazel build. + remove_grpc_version "${temp_destination_path}" +fi +###################### Section 2 ##################### +## generate gapic-*/, part of proto-*/, samples/ +###################################################### +if [[ "${proto_only}" == "false" ]]; then + "$protoc_path"/protoc --experimental_allow_proto3_optional \ + "--plugin=protoc-gen-java_gapic=${script_dir}/gapic-generator-java-wrapper" \ + "--java_gapic_out=metadata:${temp_destination_path}/java_gapic_srcjar_raw.srcjar.zip" \ + "--java_gapic_opt=$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "${gapic_yaml}" "${service_config}" "${service_yaml}")" \ + ${proto_files} ${gapic_additional_protos} + + unzip -o -q "${temp_destination_path}/java_gapic_srcjar_raw.srcjar.zip" -d "${temp_destination_path}" + # Sync'\''d to the output file name in Writer.java. + unzip -o -q "${temp_destination_path}/temp-codegen.srcjar" -d "${temp_destination_path}/java_gapic_srcjar" + # Resource name source files. + proto_dir=${temp_destination_path}/java_gapic_srcjar/proto/src/main/java + if [ ! -d "${proto_dir}" ]; then + # Some APIs don't have resource name helpers, like BigQuery v2. + # Create an empty file so we can finish building. Gating the resource name rule definition + # on file existences go against Bazel's design patterns, so we'll simply delete all empty + # files during the final packaging process (see java_gapic_pkg.bzl) + mkdir -p "${proto_dir}" + touch "${proto_dir}"/PlaceholderFile.java + fi + # move java_gapic_srcjar/src/main to gapic-*/src. + mv_src_files "gapic" "main" "${temp_destination_path}" + # remove empty files in gapic-*/src/main/java + remove_empty_files "gapic" "${temp_destination_path}" + # move java_gapic_srcjar/src/test to gapic-*/src + mv_src_files "gapic" "test" "${temp_destination_path}" + if [ "${include_samples}" == "true" ]; then + # move java_gapic_srcjar/samples/snippets to samples/snippets + mv_src_files "samples" "main" "${temp_destination_path}" + fi +fi +##################### Section 3 ##################### +# generate proto-*/ +##################################################### +# exclude certain protos to java compiler. +case "${proto_path}" in + "google/cloud/aiplatform/v1beta1"*) + # these protos are excluded from //google/cloud/aiplatform/v1beta1:google-cloud-aiplatform-v1beta1-java + prefix="google/cloud/aiplatform/v1beta1/schema" + protos="${prefix}/annotation_payload.proto ${prefix}/annotation_spec_color.proto ${prefix}/data_item_payload.proto ${prefix}/dataset_metadata.proto ${prefix}/geometry.proto" + for removed_proto in ${protos}; do + proto_files="${proto_files//${removed_proto}/}" + done + ;; +esac +"$protoc_path"/protoc "--java_out=${temp_destination_path}/java_proto.jar" ${proto_files} +if [[ "${proto_only}" == "false" ]]; then + # move java_gapic_srcjar/proto/src/main/java (generated resource name helper class) + # to proto-*/src/main + mv_src_files "proto" "main" "${temp_destination_path}" +fi +# unzip java_proto.jar to proto-*/src/main/java +unzip_src_files "proto" "${temp_destination_path}" +# remove empty files in proto-*/src/main/java +remove_empty_files "proto" "${temp_destination_path}" +case "${proto_path}" in + "google/cloud/aiplatform/v1beta1"*) + prefix="google/cloud/aiplatform/v1beta1/schema" + protos="${prefix}/annotation_payload.proto ${prefix}/annotation_spec_color.proto ${prefix}/data_item_payload.proto ${prefix}/dataset_metadata.proto ${prefix}/geometry.proto" + for added_proto in ${protos}; do + proto_files="${proto_files} ${added_proto}" + done + ;; +esac +# copy proto files to proto-*/src/main/proto +for proto_src in ${proto_files}; do + if [[ "${proto_src}" == "google/cloud/common/operation_metadata.proto" ]]; then + continue + fi + mkdir -p "${temp_destination_path}/proto-${folder_name}/src/main/proto" + rsync -R "${proto_src}" "${temp_destination_path}/proto-${folder_name}/src/main/proto" +done +popd # output_folder +##################### Section 4 ##################### +# rm tar files +##################################################### +pushd "${temp_destination_path}" +rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar +popd # destination path + +cp -r ${temp_destination_path}/* "${output_folder}/${destination_path}" +rm -rdf "${temp_destination_path}" +exit 0 diff --git a/hermetic_build/library_generation/generate_repo.py b/hermetic_build/library_generation/generate_repo.py new file mode 100755 index 0000000000..2e233acb7b --- /dev/null +++ b/hermetic_build/library_generation/generate_repo.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import shutil +from typing import Optional +import library_generation.utils.utilities as util +from library_generation.generate_composed_library import generate_composed_library +from common.model.generation_config import GenerationConfig +from common.model.library_config import LibraryConfig +from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing + + +def generate_from_yaml( + config: GenerationConfig, + repository_path: str, + api_definitions_path: str, + target_library_names: Optional[list[str]], +) -> None: + """ + Based on the generation config, generates libraries via + generate_composed_library.py + + :param config: a GenerationConfig object. + :param repository_path: The repository path to which the generated files + will be sent. + :param api_definitions_path: The path to where the api definition resides. + :param target_library_names: a list of libraries to be generated. + If specified, only the library whose library_name is in target_library_names + will be generated. + If specified with an empty list, then no library will be generated. + If not specified, all libraries in the configuration yaml will be generated. + """ + target_libraries = get_target_libraries( + config=config, target_library_names=target_library_names + ) + repo_config = util.prepare_repo( + gen_config=config, library_config=target_libraries, repo_path=repository_path + ) + # copy api definition to output folder. + shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) + + for library_path, library in repo_config.get_libraries().items(): + print(f"generating library {library.get_library_name()}") + generate_composed_library( + config=config, + library_path=library_path, + library=library, + repo_config=repo_config, + ) + + if not config.is_monorepo() or config.contains_common_protos(): + return + + monorepo_postprocessing( + repository_path=repository_path, versions_file=repo_config.versions_file + ) + + +def get_target_libraries( + config: GenerationConfig, target_library_names: list[str] = None +) -> list[LibraryConfig]: + """ + Returns LibraryConfig objects whose library_name is in target_library_names. + + :param config: a GenerationConfig object. + :param target_library_names: library_name of target libraries. + If not specified, all libraries in the given config will be returned. + :return: LibraryConfig objects. + """ + if target_library_names is None: + return config.libraries + target_libraries = set(target_library_names) + return [ + library + for library in config.libraries + if library.get_library_name() in target_libraries + ] diff --git a/hermetic_build/library_generation/model/bom_config.py b/hermetic_build/library_generation/model/bom_config.py new file mode 100644 index 0000000000..b562407eb7 --- /dev/null +++ b/hermetic_build/library_generation/model/bom_config.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BomConfig: + """ + Class that represents an entry in dependencyManagement section. + """ + + def __init__( + self, + group_id: str, + artifact_id: str, + version: str, + version_annotation: str, + is_import: bool = True, + ): + self.group_id = group_id + self.artifact_id = artifact_id + self.version = version + self.version_annotation = version_annotation + self.is_import = is_import + + def __lt__(self, another): + return self.group_id < another.group_id or ( + self.group_id == another.group_id and self.artifact_id < another.artifact_id + ) + + def __eq__(self, another): + return ( + self.group_id == another.group_id + and self.artifact_id == another.artifact_id + ) diff --git a/hermetic_build/library_generation/model/config_change.py b/hermetic_build/library_generation/model/config_change.py new file mode 100644 index 0000000000..018aee8ccd --- /dev/null +++ b/hermetic_build/library_generation/model/config_change.py @@ -0,0 +1,203 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import shutil +from enum import Enum +from typing import Optional +from git import Commit, Repo + +from common.model.gapic_inputs import parse_build_str +from common.model.generation_config import GenerationConfig +from common.model.library_config import LibraryConfig +from library_generation.utils.utilities import sh_util +from library_generation.utils.proto_path_utils import find_versioned_proto_path + +INSERTIONS = "insertions" +LINES = "lines" + + +class ChangeType(Enum): + GOOGLEAPIS_COMMIT = 1 + REPO_LEVEL_CHANGE = 2 + LIBRARIES_ADDITION = 3 + # As of Mar. 2024, we decide not to produce this type of change because we + # still need to manually remove the libray. + # LIBRARIES_REMOVAL = 4 + LIBRARY_LEVEL_CHANGE = 5 + GAPIC_ADDITION = 6 + # As of Mar. 2024, we decide not to produce this type of change because we + # still need to manually remove the libray. + # GAPIC_REMOVAL = 7 + + +class HashLibrary: + """ + Data class to group a LibraryConfig object and its hash value together. + """ + + def __init__(self, hash_value: int, library: LibraryConfig): + self.hash_value = hash_value + self.library = library + + +class LibraryChange: + def __init__(self, changed_param: str, current_value: str, library_name: str = ""): + self.changed_param = changed_param + self.current_value = current_value + self.library_name = library_name + + +class QualifiedCommit: + def __init__(self, commit: Commit, libraries: set[str]): + self.commit = commit + self.libraries = libraries + + +class ConfigChange: + ALL_LIBRARIES_CHANGED = None + + def __init__( + self, + change_to_libraries: dict[ChangeType, list[LibraryChange]], + baseline_config: GenerationConfig, + current_config: GenerationConfig, + ): + self.change_to_libraries = change_to_libraries + self.baseline_config = baseline_config + self.current_config = current_config + + def get_changed_libraries(self) -> Optional[list[str]]: + """ + Returns a unique, sorted list of library name of changed libraries. + None if there is a repository level change, which means all libraries + in the current_config will be generated. + + :return: library names of change libraries. + """ + if ChangeType.REPO_LEVEL_CHANGE in self.change_to_libraries: + return ConfigChange.ALL_LIBRARIES_CHANGED + library_names = set() + for change_type, library_changes in self.change_to_libraries.items(): + if change_type == ChangeType.GOOGLEAPIS_COMMIT: + library_names.update(self.__get_library_names_from_qualified_commits()) + else: + library_names.update( + [library_change.library_name for library_change in library_changes] + ) + return sorted(list(library_names)) + + def get_qualified_commits( + self, + repo_url: str = "https://github.com/googleapis/googleapis.git", + ) -> list[QualifiedCommit]: + """ + Returns qualified commits from configuration change. + + A qualified commit is a commit that changes at least one file (excluding + BUILD.bazel) within a versioned proto path in the given proto_paths. + :param repo_url: the repository contains the commit history. + :return: QualifiedCommit objects. + """ + tmp_dir = sh_util("get_output_folder") + shutil.rmtree(tmp_dir, ignore_errors=True) + os.mkdir(tmp_dir) + # we only need commit history, thus shadow clone is enough. + repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"]) + commit = repo.commit(self.current_config.googleapis_commitish) + proto_paths = self.current_config.get_proto_path_to_library_name() + qualified_commits = [] + while str(commit.hexsha) != self.baseline_config.googleapis_commitish: + qualified_commit = ConfigChange.__create_qualified_commit( + proto_paths=proto_paths, commit=commit + ) + if qualified_commit is not None: + qualified_commits.append(qualified_commit) + commit_parents = commit.parents + if len(commit_parents) == 0: + break + commit = commit_parents[0] + shutil.rmtree(tmp_dir, ignore_errors=True) + return qualified_commits + + def __get_library_names_from_qualified_commits(self) -> list[str]: + qualified_commits = self.get_qualified_commits() + library_names = [] + for qualified_commit in qualified_commits: + library_names.extend(qualified_commit.libraries) + return library_names + + @staticmethod + def __create_qualified_commit( + proto_paths: dict[str, str], commit: Commit + ) -> Optional[QualifiedCommit]: + """ + Returns a qualified commit from the given Commit object; otherwise None. + + :param proto_paths: a mapping from versioned proto_path to library_name + :param commit: a GitHub commit object. + :return: qualified commits. + """ + libraries = set() + for file_path, changes in commit.stats.files.items(): + versioned_proto_path = find_versioned_proto_path(file_path) + if versioned_proto_path in proto_paths: + if ( + file_path.endswith("BUILD.bazel") + # Qualify a commit if the commit only added BUILD.bazel + # because it's very unlikely that a commit added BUILD.bazel + # without adding proto files. Therefore, the commit is + # qualified duo to the proto change eventually. + and (not ConfigChange.__is_added(changes)) + and ( + not ConfigChange.__is_qualified_build_change( + commit=commit, build_file_path=file_path + ) + ) + ): + continue + # Even though a commit usually only changes one + # library, we don't want to miss generating a + # library because the commit may change multiple + # libraries. + libraries.add(proto_paths[versioned_proto_path]) + if len(libraries) == 0: + return None + return QualifiedCommit(commit=commit, libraries=libraries) + + @staticmethod + def __is_added(changes: dict[str, int]) -> bool: + return changes[INSERTIONS] == changes[LINES] + + @staticmethod + def __is_qualified_build_change(commit: Commit, build_file_path: str) -> bool: + """ + Checks if the given commit containing a BUILD.bazel change is a + qualified commit. + + The commit is a qualified commit if the + :class:`library_generation.model.gapic_inputs.GapicInputs` objects + parsed from the commit and its parent are different, since there are + changes in fields that used in library generation. + + :param commit: a GitHub commit object. + :param build_file_path: the path of the BUILD.bazel + :return: True if the commit is a qualified commit; False otherwise. + """ + versioned_proto_path = find_versioned_proto_path(build_file_path) + build = str((commit.tree / build_file_path).data_stream.read()) + parent_commit = commit.parents[0] + parent_build = str((parent_commit.tree / build_file_path).data_stream.read()) + inputs = parse_build_str(build, versioned_proto_path) + parent_inputs = parse_build_str(parent_build, versioned_proto_path) + return inputs != parent_inputs diff --git a/hermetic_build/library_generation/model/repo_config.py b/hermetic_build/library_generation/model/repo_config.py new file mode 100644 index 0000000000..1921b177d9 --- /dev/null +++ b/hermetic_build/library_generation/model/repo_config.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from common.model.library_config import LibraryConfig + + +GRPC_PREFIX = "grpc-" +PROTO_PREFIX = "proto-" +NEW_CLIENT_VERSION = "0.0.0" + + +class RepoConfig: + """ + Class that represents a generated repository + """ + + def __init__( + self, + output_folder: str, + libraries: dict[str, LibraryConfig], + versions_file: str, + ): + """ + Init a RepoConfig object + + :param output_folder: the path to which the generated repo goes + :param libraries: a mapping from library_path to LibraryConfig object + :param versions_file: the path of versions.txt used in post-processing + """ + self.output_folder = output_folder + self.libraries = libraries + self.versions_file = versions_file + self.library_versions = self.__parse_versions() + + def get_libraries(self) -> dict[str, LibraryConfig]: + return self.libraries + + def get_library_version(self, artifact_id: str) -> str: + """ + Returns the version of a given artifact ID. + If the artifact ID is not managed, i.e., a new client, returns `0.0.0`. + + :param artifact_id: the Maven artifact ID. + :return: the version of the artifact. + """ + return self.library_versions.get(artifact_id, NEW_CLIENT_VERSION) + + def __parse_versions(self) -> dict[str, str]: + """ + For a given versions.txt file (defined in self.versions_file) + creates a map of artifact-id to its version + + :return: a map "artifact-id -> version" + """ + library_versions = dict() + with open(self.versions_file) as f: + for line in f.readlines(): + sections = line.split(":") + # skip comments and empty lines. + if len(sections) != 3: + continue + artifact_id = sections[0] + released_version = sections[1] + if artifact_id.startswith(GRPC_PREFIX) or artifact_id.startswith( + PROTO_PREFIX + ): + continue + library_versions[artifact_id] = released_version + return library_versions diff --git a/hermetic_build/library_generation/owlbot/bin/entrypoint.sh b/hermetic_build/library_generation/owlbot/bin/entrypoint.sh new file mode 100755 index 0000000000..3d38677678 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/bin/entrypoint.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is the entrypoint script for java owlbot. This is not intended to be +# called directly but rather be called from postproces_library.sh +# For reference, the positional arguments are +# 1: scripts_root: location of the root of the library_generation scripts. When +# in a Docker container, this value should be /src/ +# 2: versions_file: points to a versions.txt containing versions to be applied +# both to README and pom.xml files +# 3: is_monorepo: whether we are postprocessing a monorepo +# 4: libraries_bom_version: used to render the readme +# 5: library_version: used to render the readme + +# The scripts assumes the CWD is the folder where postprocessing is going to be +# applied + +set -ex +scripts_root=$1 +versions_file=$2 +is_monorepo=$3 +libraries_bom_version=$4 +library_version=$5 + +if [[ "${is_monorepo}" == "true" ]]; then + mv owl-bot-staging/* temp + rm -rd owl-bot-staging/ + mv temp owl-bot-staging +fi + +# templates as well as retrieving files from owl-bot-staging +echo "Retrieving files from owl-bot-staging directory..." +if [ -f "owlbot.py" ] +then + # we copy the templates to a temp folder because we need to do a special + # modification regarding libraries_bom_version that can't be handled by the + # synthtool library considering the way owlbot.py files are written + export SYNTHTOOL_TEMPLATES="${scripts_root}/owlbot/templates" + export SYNTHTOOL_LIBRARIES_BOM_VERSION="${libraries_bom_version}" + export SYNTHTOOL_LIBRARY_VERSION="${library_version}" + # defaults to run owlbot.py + python3 owlbot.py + unset SYNTHTOOL_TEMPLATES + unset SYNTHTOOL_LIBRARIES_BOM_VERSION + unset SYNTHTOOL_LIBRARY_VERSION +fi +echo "...done" + +# write or restore pom.xml files +echo "Generating missing pom.xml..." +python3 "${scripts_root}/owlbot/src/fix_poms.py" "${versions_file}" "${is_monorepo}" +echo "...done" + +# write or restore clirr-ignored-differences.xml +echo "Generating clirr-ignored-differences.xml..." +"${scripts_root}"/owlbot/bin/write_clirr_ignore.sh "${scripts_root}" +echo "...done" + +# fix license headers +echo "Fixing missing license headers..." +python3 "${scripts_root}/owlbot/src/fix-license-headers.py" +echo "...done" + +echo "Reformatting source..." +"${scripts_root}"/owlbot/bin/format_source.sh "${scripts_root}" +echo "...done" diff --git a/hermetic_build/library_generation/owlbot/bin/format_source.sh b/hermetic_build/library_generation/owlbot/bin/format_source.sh new file mode 100755 index 0000000000..9402be778b --- /dev/null +++ b/hermetic_build/library_generation/owlbot/bin/format_source.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Why OwlBot Java postprocessor does not use the formatter defined in pom.xml? +# It's because the postprocessor runs in a privileged (albeit limited) +# environment. We limit the risk of running somebody else's malicious Maven +# plugin code in the environment. + +# Find all the java files relative to the current directory and format them +# using google-java-format +scripts_root=$1 + +source "${scripts_root}/utils/utilities.sh" +list="$(find . -name '*.java' -not -path ".*/samples/snippets/generated/**/*" )" +tmp_file=$(mktemp) + +for file in $list; +do + if [[ $file =~ .*/samples/snippets/src/main/java/com/example/firestore/Quickstart.java ]]; + then + echo "File skipped formatting: $file" + elif [[ $file =~ .*/samples/snippets/src/.*/java/com/example/spanner/.*.java ]]; + then + echo "File skipped formatting: $file" + else + echo $file >> $tmp_file + fi +done + +# use formatter downloaded in the Dockerfile. +cat $tmp_file | xargs java -jar "$(get_java_formatter_location)" --replace + +rm $tmp_file diff --git a/hermetic_build/library_generation/owlbot/bin/write_clirr_ignore.sh b/hermetic_build/library_generation/owlbot/bin/write_clirr_ignore.sh new file mode 100755 index 0000000000..d6925ef354 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/bin/write_clirr_ignore.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +scripts_root=$1 +templates_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")/../templates/clirr) + +# restore default clirr-ignored-differences.xml for protos if the file does not exist +for dir in `ls -d proto-google-*` +do + if [ ! -f "${dir}/clirr-ignored-differences.xml" ] + then + tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX) + pushd ${dir} + pushd src/main/java + find * -name *OrBuilder.java | xargs dirname | sort -u | jq -Rns ' (inputs | rtrimstr("\n") | split("\n") ) as $data | {proto_paths: $data}' > ${tmp_dir}/paths.json + popd + python3 "${scripts_root}/owlbot/src/gen-template.py" --data=${tmp_dir}/paths.json --folder=${templates_dir} + popd + fi +done diff --git a/hermetic_build/library_generation/owlbot/src/fix-license-headers.py b/hermetic_build/library_generation/owlbot/src/fix-license-headers.py new file mode 100644 index 0000000000..1c690b5d39 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/src/fix-license-headers.py @@ -0,0 +1,30 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import glob +from synthtool.languages import java + + +root = Path(".").resolve() + +# Until the generator generates license headers on generated proto +# classes, add the license headers in +for path in glob.glob("proto-*"): + java.fix_proto_headers(root / path) + +# Until the generator generates license headers on generated grpc +# classes, add the license headers in +for path in glob.glob("grpc-*"): + java.fix_grpc_headers(root / path, "unused") diff --git a/hermetic_build/library_generation/owlbot/src/fix_poms.py b/hermetic_build/library_generation/owlbot/src/fix_poms.py new file mode 100644 index 0000000000..9ee7514a4d --- /dev/null +++ b/hermetic_build/library_generation/owlbot/src/fix_poms.py @@ -0,0 +1,611 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import glob +import json +from xml.etree.ElementTree import ElementTree + +from lxml import etree +import os +import re +from typing import List, Mapping +from library_generation.owlbot.src.poms import module, templates + + +def load_versions(filename: str, default_group_id: str) -> Mapping[str, module.Module]: + if not os.path.isfile(filename): + return {} + modules = {} + with open(filename, "r") as fp: + for line in fp: + line = line.strip() + if line.startswith("#"): + continue + + parts = line.split(":") + if len(parts) == 3: + artifact_id = parts[0] + group_id = ( + # For artifact id starts with `proto-` or `grpc-`, we + # need special treatments to append `.api.grpc` suffix + # to its corresponding group id. + # For other artifact id, keep the existing group id. + # Other than the two aforementioned artifact id, do not + # assume artifact id always starts with `google-`. Known + # exception is ad-manager. + __proto_group_id(default_group_id) + if artifact_id.startswith("proto-") + or artifact_id.startswith("grpc-") + else default_group_id + ) + modules[artifact_id] = module.Module( + group_id=group_id, + artifact_id=artifact_id, + release_version=parts[1], + version=parts[2], + ) + return modules + + +def _find_dependency_index(dependencies, group_id, artifact_id) -> int: + try: + return next( + i + for i, x in enumerate(dependencies.getchildren()) + if _dependency_matches(x, group_id, artifact_id) + ) + except StopIteration: + return -1 + + +def _dependency_matches(node, group_id, artifact_id) -> bool: + artifact_node = node.find("{http://maven.apache.org/POM/4.0.0}artifactId") + group_node = node.find("{http://maven.apache.org/POM/4.0.0}groupId") + + if artifact_node is None or group_node is None: + return False + + return artifact_node.text.startswith(artifact_id) and group_node.text.startswith( + group_id + ) + + +def _is_cloud_client(existing_modules: List[module.Module]) -> bool: + proto_modules_len = 0 + grpc_modules_len = 0 + for artifact in existing_modules: + if artifact.startswith("proto-"): + proto_modules_len += 1 + if artifact.startswith("grpc-"): + grpc_modules_len += 1 + return proto_modules_len > 0 or grpc_modules_len > 0 + + +def update_cloud_pom( + filename: str, + proto_modules: List[module.Module], + grpc_modules: List[module.Module], + is_monorepo: bool, +): + tree = etree.parse(filename) + root = tree.getroot() + dependencies = root.find("{http://maven.apache.org/POM/4.0.0}dependencies") + + existing_dependencies = [ + m.find("{http://maven.apache.org/POM/4.0.0}artifactId").text + for m in dependencies + if m.find("{http://maven.apache.org/POM/4.0.0}artifactId") is not None + ] + + if is_monorepo: + _set_test_scoped_deps(dependencies) + + try: + grpc_index = _find_dependency_index( + dependencies, "com.google.api.grpc", "grpc-" + ) + except StopIteration: + grpc_index = _find_dependency_index(dependencies, "junit", "junit") + # insert grpc dependencies after junit + for m in grpc_modules: + if m.artifact_id not in existing_dependencies: + print(f"adding new test dependency {m.artifact_id}") + new_dependency = etree.Element( + "{http://maven.apache.org/POM/4.0.0}dependency" + ) + new_dependency.tail = "\n " + new_dependency.text = "\n " + new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") + new_group.text = m.group_id + new_group.tail = "\n " + new_artifact = etree.Element( + "{http://maven.apache.org/POM/4.0.0}artifactId" + ) + new_artifact.text = m.artifact_id + new_artifact.tail = "\n " + new_scope = etree.Element("{http://maven.apache.org/POM/4.0.0}scope") + new_scope.text = "test" + new_scope.tail = "\n " + new_dependency.append(new_group) + new_dependency.append(new_artifact) + new_dependency.append(new_scope) + dependencies.insert(grpc_index + 1, new_dependency) + + try: + proto_index = _find_dependency_index( + dependencies, "com.google.api.grpc", "proto-" + ) + except StopIteration: + print("after protobuf") + proto_index = _find_dependency_index( + dependencies, "com.google.protobuf", "protobuf-java" + ) + # insert proto dependencies after protobuf-java + for m in proto_modules: + if m.artifact_id not in existing_dependencies: + if re.match(r"proto-.*-v\d+.*", m.artifact_id): + print(f"adding new dependency {m.artifact_id}") + new_dependency = etree.Element( + "{http://maven.apache.org/POM/4.0.0}dependency" + ) + new_dependency.tail = "\n " + new_dependency.text = "\n " + new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") + new_group.text = m.group_id + new_group.tail = "\n " + new_artifact = etree.Element( + "{http://maven.apache.org/POM/4.0.0}artifactId" + ) + new_artifact.text = m.artifact_id + new_artifact.tail = "\n " + new_dependency.append(new_group) + new_dependency.append(new_artifact) + dependencies.insert(proto_index + 1, new_dependency) + + tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") + + +def _set_test_scoped_deps(dependencies: list[ElementTree]) -> None: + """ + As of July 2024, we have two dependencies that should be declared as + test-scoped in a monorepo: grpc-google-common-protos and grpc-google-iam-v1. + HW libraries are treated as usual + :param dependencies: List of XML Objects representing a + """ + TEST_SCOPED_DEPENDENCIES = ["grpc-google-common-protos", "grpc-google-iam-v1"] + print( + 'converting dependencies "grpc-google-common-protos" and "grpc-google-iam-v1" to test-scoped' + ) + for d in dependencies: + artifact_query = "{http://maven.apache.org/POM/4.0.0}artifactId" + scope_query = "{http://maven.apache.org/POM/4.0.0}scope" + current_scope = d.find(scope_query) + artifact_id_elem = d.find(artifact_query) + if artifact_id_elem is None: + continue + artifact_id = artifact_id_elem.text + is_test_scoped = ( + current_scope.text == "test" if current_scope is not None else False + ) + if artifact_id in TEST_SCOPED_DEPENDENCIES and not is_test_scoped: + new_scope = etree.Element(scope_query) + new_scope.text = "test" + if current_scope is not None: + d.replace(current_scope, new_scope) + else: + d.append(new_scope) + new_scope.tail = "\n " + new_scope.getprevious().tail = "\n " + + +def update_parent_pom(filename: str, modules: List[module.Module]): + tree = etree.parse(filename) + root = tree.getroot() + + # BEGIN: update modules + existing = root.find("{http://maven.apache.org/POM/4.0.0}modules") + + module_names = [m.artifact_id for m in modules] + extra_modules = [ + m.text for i, m in enumerate(existing) if m.text not in module_names + ] + + modules_to_write = module_names + extra_modules + num_modules = len(modules_to_write) + + existing.clear() + existing.text = "\n " + for index, m in enumerate(modules_to_write): + new_module = etree.Element("{http://maven.apache.org/POM/4.0.0}module") + new_module.text = m + if index == num_modules - 1: + new_module.tail = "\n " + else: + new_module.tail = "\n " + existing.append(new_module) + + existing.tail = "\n\n " + # END: update modules + + # BEGIN: update versions in dependencyManagement + dependencies = root.find( + "{http://maven.apache.org/POM/4.0.0}dependencyManagement" + ).find("{http://maven.apache.org/POM/4.0.0}dependencies") + + existing_dependencies = [ + m.find("{http://maven.apache.org/POM/4.0.0}artifactId").text + for m in dependencies + if m.find("{http://maven.apache.org/POM/4.0.0}artifactId") is not None + ] + insert_index = 1 + + num_modules = len(modules) + + for index, m in enumerate(modules): + if m.artifact_id in existing_dependencies: + continue + + new_dependency = etree.Element("{http://maven.apache.org/POM/4.0.0}dependency") + new_dependency.tail = "\n " + new_dependency.text = "\n " + new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") + new_group.text = m.group_id + new_group.tail = "\n " + new_artifact = etree.Element("{http://maven.apache.org/POM/4.0.0}artifactId") + new_artifact.text = m.artifact_id + new_artifact.tail = "\n " + new_version = etree.Element("{http://maven.apache.org/POM/4.0.0}version") + new_version.text = m.version + comment = etree.Comment(" {x-version-update:" + m.artifact_id + ":current} ") + comment.tail = "\n " + new_dependency.append(new_group) + new_dependency.append(new_artifact) + new_dependency.append(new_version) + new_dependency.append(comment) + new_dependency.tail = "\n " + dependencies.insert(1, new_dependency) + + # END: update versions in dependencyManagement + + tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") + + +def update_bom_pom(filename: str, modules: List[module.Module]): + tree = etree.parse(filename) + root = tree.getroot() + existing = root.find( + "{http://maven.apache.org/POM/4.0.0}dependencyManagement" + ).find("{http://maven.apache.org/POM/4.0.0}dependencies") + + num_modules = len(modules) + + existing.clear() + existing.text = "\n " + for index, m in enumerate(modules): + new_dependency = etree.Element("{http://maven.apache.org/POM/4.0.0}dependency") + new_dependency.tail = "\n " + new_dependency.text = "\n " + new_group = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") + new_group.text = m.group_id + new_group.tail = "\n " + new_artifact = etree.Element("{http://maven.apache.org/POM/4.0.0}artifactId") + new_artifact.text = m.artifact_id + new_artifact.tail = "\n " + new_version = etree.Element("{http://maven.apache.org/POM/4.0.0}version") + new_version.text = m.version + comment = etree.Comment(" {x-version-update:" + m.artifact_id + ":current} ") + comment.tail = "\n " + new_dependency.append(new_group) + new_dependency.append(new_artifact) + new_dependency.append(new_version) + new_dependency.append(comment) + + if index == num_modules - 1: + new_dependency.tail = "\n " + else: + new_dependency.tail = "\n " + existing.append(new_dependency) + + existing.tail = "\n " + + tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") + + +# When generating non-cloud client library, the group id of proto/grpc artifacts +# is prefixed with `{main_artifact_group_id}.api.grpc`, rather than +# `com.google.api.grpc`. +# https://github.com/googleapis/google-cloud-java/issues/9125 +# However, some exceptions are com.google.area120 and com.google.analytics. +# https://github.com/googleapis/google-cloud-java/issues/9304 +def __proto_group_id(main_artifact_group_id: str) -> str: + prefix = "com.google" + list_of_group_id = [ + "com.google.cloud", + "com.google.area120", + "com.google.analytics", + ] + if main_artifact_group_id not in list_of_group_id: + prefix = main_artifact_group_id + return f"{prefix}.api.grpc" + + +def __get_monorepo_version(versions: str) -> str: + """ + Returns the current version of google-cloud-java in the given version file + :param versions: the versions.txt + :return: the current version of google-cloud-java + """ + with open(versions, "r") as f: + for line in f.readlines(): + if "google-cloud-java" in line: + return line.split(":")[-1].strip() + + +def main(versions_file, monorepo): + print(f"working directory: {os.getcwd()}") + with open(".repo-metadata.json", "r") as fp: + repo_metadata = json.load(fp) + group_id, artifact_id = repo_metadata["distribution_name"].split(":") + name = repo_metadata["name_pretty"] + existing_modules = load_versions(versions_file, group_id) + print(f"monorepo? {monorepo}") + + # extra modules that need to be manages in versions.txt + if "extra_versioned_modules" in repo_metadata: + extra_managed_modules = repo_metadata["extra_versioned_modules"].split(",") + else: + extra_managed_modules = "" + + # list of modules to be excluded from added to poms + if "excluded_dependencies" in repo_metadata: + excluded_dependencies_list = repo_metadata["excluded_dependencies"].split(",") + else: + excluded_dependencies_list = "" + + # list of poms that have to be excluded from post processing + if "excluded_poms" in repo_metadata: + excluded_poms_list = repo_metadata["excluded_poms"].split(",") + else: + excluded_poms_list = "" + + # Missing Case 1: When this library ('java-XXX' module) is new. + if artifact_id not in existing_modules: + existing_modules[artifact_id] = module.Module( + group_id=group_id, + artifact_id=artifact_id, + version="0.0.1-SNAPSHOT", + release_version="0.0.0", + ) + main_module = existing_modules[artifact_id] + + # Artifact ID is part of distribution name field in .repo-metadata.json + if artifact_id in [ + "grafeas", + "google-cloud-dns", + "google-cloud-notification", + "google-iam-policy", + ]: + # There are special libraries that are not automatically generated + print( + f"Skipping a special case library {artifact_id} that do not have " + " the standard module structure." + ) + return + + parent_artifact_id = f"{artifact_id}-parent" + + if parent_artifact_id not in existing_modules: + existing_modules[parent_artifact_id] = module.Module( + group_id=group_id, + artifact_id=parent_artifact_id, + version=main_module.version, + release_version=main_module.release_version, + ) + parent_module = existing_modules[parent_artifact_id] + + required_dependencies = {} + for dependency_module in existing_modules: + if dependency_module in excluded_dependencies_list: + continue + dep_artifact_id = existing_modules[dependency_module].artifact_id + if monorepo and not os.path.isdir(dep_artifact_id): + # In monorepo, existing_modules are loaded from the root + # versions.txt and thus includes irrelevant artifacts + continue + required_dependencies[dependency_module] = existing_modules[dependency_module] + + # Missing Case 2: There's a new proto-XXX and grpc-XXX directory. It's a new + # version in the proto file to a library. Both a new library and existing + # library. + for path in glob.glob("proto-*"): + if not path in existing_modules: + existing_modules[path] = module.Module( + group_id=__proto_group_id(group_id), + artifact_id=path, + version=main_module.version, + release_version=main_module.release_version, + ) + if ( + path not in excluded_dependencies_list + and path not in main_module.artifact_id + ): + required_dependencies[path] = module.Module( + group_id=__proto_group_id(group_id), + artifact_id=path, + version=main_module.version, + release_version=main_module.release_version, + ) + if not os.path.isfile(f"{path}/pom.xml"): + print(f"creating missing proto pom: {path}") + templates.render( + template_name="proto_pom.xml.j2", + output_name=f"{path}/pom.xml", + module=required_dependencies[path], + parent_module=parent_module, + main_module=main_module, + monorepo=monorepo, + ) + if ( + path not in excluded_dependencies_list + and path not in main_module.artifact_id + ): + required_dependencies[path] = module.Module( + group_id=__proto_group_id(group_id), + artifact_id=path, + version=main_module.version, + release_version=main_module.release_version, + ) + + for path in glob.glob("grpc-*"): + if not path in existing_modules: + existing_modules[path] = module.Module( + group_id=__proto_group_id(group_id), + artifact_id=path, + version=main_module.version, + release_version=main_module.release_version, + ) + if ( + path not in excluded_dependencies_list + and path not in main_module.artifact_id + ): + required_dependencies[path] = module.Module( + group_id=__proto_group_id(group_id), + artifact_id=path, + version=main_module.version, + release_version=main_module.release_version, + ) + + if not os.path.isfile(f"{path}/pom.xml"): + proto_artifact_id = path.replace("grpc-", "proto-") + print(f"creating missing grpc pom: {path}") + templates.render( + template_name="grpc_pom.xml.j2", + output_name=f"{path}/pom.xml", + module=required_dependencies[path], + parent_module=parent_module, + main_module=main_module, + proto_module=existing_modules[proto_artifact_id], + monorepo=monorepo, + ) + if ( + path not in excluded_dependencies_list + and path not in main_module.artifact_id + ): + required_dependencies[path] = module.Module( + group_id=__proto_group_id(group_id), + artifact_id=path, + version=main_module.version, + release_version=main_module.release_version, + ) + proto_modules = [ + module + for module in required_dependencies.values() + if module.artifact_id.startswith("proto-") + and module.artifact_id not in parent_artifact_id + ] + grpc_modules = [ + module + for module in required_dependencies.values() + if module.artifact_id.startswith("grpc-") + and module.artifact_id not in parent_artifact_id + ] + if main_module in grpc_modules or main_module in proto_modules: + modules = grpc_modules + proto_modules + else: + modules = [main_module] + grpc_modules + proto_modules + + if not _is_cloud_client(existing_modules): + print("no proto or grpc modules - probably not a cloud client") + return + + if os.path.isfile(f"{artifact_id}/pom.xml"): + print("updating modules in cloud pom.xml") + if artifact_id not in excluded_poms_list: + update_cloud_pom( + f"{artifact_id}/pom.xml", proto_modules, grpc_modules, monorepo + ) + elif artifact_id not in excluded_poms_list: + print("creating missing cloud pom.xml") + templates.render( + template_name="cloud_pom.xml.j2", + output_name=f"{artifact_id}/pom.xml", + module=main_module, + parent_module=parent_module, + repo=repo_metadata["repo"], + name=name, + description=repo_metadata["api_description"], + proto_modules=proto_modules, + grpc_modules=grpc_modules, + monorepo=monorepo, + ) + + if os.path.isfile(f"{artifact_id}-bom/pom.xml"): + print("updating modules in bom pom.xml") + if artifact_id + "-bom" not in excluded_poms_list: + update_bom_pom(f"{artifact_id}-bom/pom.xml", modules) + elif artifact_id + "-bom" not in excluded_poms_list: + print("creating missing bom pom.xml") + monorepo_version = __get_monorepo_version(versions_file) if monorepo else "" + templates.render( + template_name="bom_pom.xml.j2", + output_name=f"{artifact_id}-bom/pom.xml", + repo=repo_metadata["repo"], + name=name, + modules=modules, + main_module=main_module, + monorepo=monorepo, + monorepo_version=monorepo_version, + ) + + if os.path.isfile("pom.xml"): + print("updating modules in parent pom.xml") + update_parent_pom("pom.xml", modules) + else: + print("creating missing parent pom.xml") + monorepo_version = __get_monorepo_version(versions_file) if monorepo else "" + templates.render( + template_name="parent_pom.xml.j2", + output_name="./pom.xml", + repo=repo_metadata["repo"], + modules=modules, + main_module=main_module, + name=name, + monorepo=monorepo, + monorepo_version=monorepo_version, + ) + + print(f"updating modules in {versions_file}") + existing_modules.pop(parent_artifact_id) + + # add extra modules to versions.txt + for dependency_module in extra_managed_modules: + if dependency_module not in existing_modules: + existing_modules[dependency_module] = module.Module( + group_id=__proto_group_id(group_id), + artifact_id=dependency_module, + version=main_module.version, + release_version=main_module.release_version, + ) + templates.render( + template_name="versions.txt.j2", + output_name=versions_file, + modules=existing_modules.values(), + ) + + +if __name__ == "__main__": + versions_file = sys.argv[1] + monorepo = True if sys.argv[2].lower() == "true" else False + main(versions_file, monorepo) diff --git a/hermetic_build/library_generation/owlbot/src/gen-template.py b/hermetic_build/library_generation/owlbot/src/gen-template.py new file mode 100644 index 0000000000..30f8c43529 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/src/gen-template.py @@ -0,0 +1,84 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +from typing import List +import os +from pathlib import Path + +import click +import jinja2 + + +@click.command() +@click.option( + "--folder", + help="Path to folder of templates", +) +@click.option("--file", help="Path to template file") +@click.option( + "--data", + help="Path to JSON file with template values", + multiple=True, + required=True, +) +@click.option( + "--output", + help="Path to output", + default=".", +) +def main(folder: str, file: str, data: List[str], output: str): + """Generate templates""" + variables = {} + for data_file in data: + with open(data_file, "r") as fp: + variables = {**variables, **json.load(fp)} + + if folder is not None: + location = Path(folder) + filenames = glob.glob(f"{folder}/**/*.j2", recursive=True) + elif file is not None: + location = Path(file).parent + filenames = [f"{file}.j2"] + else: + raise Exception("Need to specify either folder or file") + + output_path = Path(output) + + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(str(location)), + autoescape=False, + keep_trailing_newline=True, + ) + + for filename in filenames: + template_name = Path(filename).relative_to(location) + template = env.get_template(str(template_name)) + output = template.stream(**variables) + + destination = output_path / os.path.splitext(template_name)[0] + destination.parent.mkdir(parents=True, exist_ok=True) + + with destination.open("w") as fp: + output.dump(fp) + + # Copy file mode over + source_path = Path(template.filename) + mode = source_path.stat().st_mode + destination.chmod(mode) + + +if __name__ == "__main__": + main() diff --git a/hermetic_build/library_generation/owlbot/src/poms/.gitignore b/hermetic_build/library_generation/owlbot/src/poms/.gitignore new file mode 100644 index 0000000000..c18dd8d83c --- /dev/null +++ b/hermetic_build/library_generation/owlbot/src/poms/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/hermetic_build/library_generation/owlbot/src/poms/module.py b/hermetic_build/library_generation/owlbot/src/poms/module.py new file mode 100644 index 0000000000..b4274ea969 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/src/poms/module.py @@ -0,0 +1,54 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import attr +from lxml import etree +import os +from typing import List, Optional + + +@attr.s(auto_attribs=True) +class Module: + group_id: str + artifact_id: str + version: str + release_version: Optional[str] + + +def read_module(pom: str) -> Module: + tree = etree.parse(pom) + artifact_id = tree.find("{http://maven.apache.org/POM/4.0.0}artifactId").text + version = tree.find("{http://maven.apache.org/POM/4.0.0}version").text + group_id = ( + "com.google.cloud" + if artifact_id.startswith("google-cloud") + else "com.google.api.grpc" + ) + return Module( + group_id=group_id, + artifact_id=artifact_id, + version=version, + ) + + +def read_modules(service: str) -> List[Module]: + thedir = f"workspace/java-{service}/" + modules = [] + for name in os.listdir(thedir): + dir = os.path.join(thedir, name) + pom = os.path.join(dir, "pom.xml") + if os.path.exists(pom): + modules.append(read_module(pom)) + + return modules diff --git a/hermetic_build/library_generation/owlbot/src/poms/templates.py b/hermetic_build/library_generation/owlbot/src/poms/templates.py new file mode 100644 index 0000000000..d100a2bc5d --- /dev/null +++ b/hermetic_build/library_generation/owlbot/src/poms/templates.py @@ -0,0 +1,35 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from jinja2 import Environment, FileSystemLoader +import os +import pathlib + +root_directory = pathlib.Path( + os.path.realpath(os.path.dirname(os.path.realpath(__file__))) +).parent.parent +print(root_directory) +jinja_env = Environment( + loader=FileSystemLoader(str(root_directory / "templates" / "poms")), + keep_trailing_newline=True, +) + + +def render(template_name: str, output_name: str, **kwargs): + template = jinja_env.get_template(template_name) + t = template.stream(kwargs) + directory = os.path.dirname(output_name) + if not os.path.isdir(directory): + os.makedirs(directory) + t.dump(str(output_name)) diff --git a/hermetic_build/library_generation/owlbot/synthtool/__init__.py b/hermetic_build/library_generation/owlbot/synthtool/__init__.py new file mode 100644 index 0000000000..e2d44deaf0 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/synthtool/__init__.py @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Synthtool synthesizes libraries from disparate sources.""" + +from synthtool.transforms import ( + move, + replace, + get_staging_dirs, + remove_staging_dirs, +) + +copy = move + +__all__ = [ + "copy", + "move", + "replace", + "get_staging_dirs", + "remove_staging_dirs", +] diff --git a/hermetic_build/library_generation/owlbot/synthtool/_tracked_paths.py b/hermetic_build/library_generation/owlbot/synthtool/_tracked_paths.py new file mode 100644 index 0000000000..b72d7037fe --- /dev/null +++ b/hermetic_build/library_generation/owlbot/synthtool/_tracked_paths.py @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tracked paths. + +This is a bit of a hack (imported from original synthtool library). +""" + +import pathlib + + +_tracked_paths = [] + + +def add(path): + _tracked_paths.append(pathlib.Path(path)) + # Reverse sort the list, so that the deepest paths get matched first. + _tracked_paths.sort(key=lambda s: -len(str(s))) + + +def relativize(path): + path = pathlib.Path(path) + for tracked_path in _tracked_paths: + try: + return path.relative_to(tracked_path) + except ValueError: + pass + raise ValueError(f"The root for {path} is not tracked.") diff --git a/hermetic_build/library_generation/owlbot/synthtool/gcp/common.py b/hermetic_build/library_generation/owlbot/synthtool/gcp/common.py new file mode 100644 index 0000000000..5cd648be36 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/synthtool/gcp/common.py @@ -0,0 +1,146 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import sys +import yaml +from pathlib import Path +from typing import Dict, List, Optional +import logging + +from synthtool import _tracked_paths +from synthtool.sources import templates + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) +# Originally brought from gcp/partials.py. +# These are the default locations to look up +_DEFAULT_PARTIAL_FILES = [ + ".readme-partials.yml", + ".readme-partials.yaml", + ".integration-partials.yaml", +] + + +class CommonTemplates: + def __init__(self, template_path: Optional[Path] = None): + local_templates = os.environ.get("SYNTHTOOL_TEMPLATES") + if local_templates is None: + logger.error("env var SYNTHTOOL_TEMPLATES must be set") + sys.exit(1) + logger.debug(f"Using local templates at {local_templates}") + self._template_root = Path(local_templates) + self._templates = templates.Templates(self._template_root) + self.excludes = [] # type: List[str] + + def _generic_library(self, directory: str, relative_dir=None, **kwargs) -> Path: + # load common repo meta information (metadata that's not language specific). + if "metadata" in kwargs: + self._load_generic_metadata(kwargs["metadata"], relative_dir=relative_dir) + # if no samples were found, don't attempt to render a + # samples/README.md. + if "samples" not in kwargs["metadata"] or not kwargs["metadata"]["samples"]: + self.excludes.append("samples/README.md") + + t = templates.TemplateGroup(self._template_root / directory, self.excludes) + + result = t.render(**kwargs) + _tracked_paths.add(result) + + return result + + def java_library(self, **kwargs) -> Path: + # kwargs["metadata"] is required to load values from .repo-metadata.json + if "metadata" not in kwargs: + kwargs["metadata"] = {} + return self._generic_library("java_library", **kwargs) + + def render(self, template_name: str, **kwargs) -> Path: + template = self._templates.render(template_name, **kwargs) + _tracked_paths.add(template) + return template + + def _load_generic_metadata(self, metadata: Dict, relative_dir=None): + """ + loads additional meta information from .repo-metadata.json. + """ + metadata["partials"] = load_partials() + + # Loads repo metadata information from the default location if it + # hasn't already been set. Some callers may have already loaded repo + # metadata, so we don't need to do it again or overwrite it. Also, only + # set the "repo" key. + if "repo" not in metadata: + metadata["repo"] = _load_repo_metadata(relative_dir=relative_dir) + + +def _load_repo_metadata( + relative_dir=None, metadata_file: str = "./.repo-metadata.json" +) -> Dict: + """Parse a metadata JSON file into a Dict. + + Currently, the defined fields are: + * `name` - The service's API name + * `name_pretty` - The service's API title. This will be used for generating titles on READMEs + * `product_documentation` - The product documentation on cloud.google.com + * `client_documentation` - The client library reference documentation + * `issue_tracker` - The public issue tracker for the product + * `release_level` - The release level of the client library. One of: alpha, beta, + ga, deprecated, preview, stable + * `language` - The repo language. One of dotnet, go, java, nodejs, php, python, ruby + * `repo` - The GitHub repo in the format {owner}/{repo} + * `distribution_name` - The language-idiomatic package/distribution name + * `api_id` - The API ID associated with the service. Fully qualified identifier use to + enable a service in the cloud platform (e.g. monitoring.googleapis.com) + * `requires_billing` - Whether or not the API requires billing to be configured on the + customer's acocunt + + Args: + metadata_file (str, optional): Path to the metadata json file + + Returns: + A dictionary of metadata. This may not necessarily include all the defined fields above. + """ + if relative_dir is not None: + if os.path.exists(Path(relative_dir, metadata_file).resolve()): + with open(Path(relative_dir, metadata_file).resolve()) as f: + return json.load(f) + elif os.path.exists(metadata_file): + with open(metadata_file) as f: + return json.load(f) + return {} + + +def load_partials(files: List[str] = []) -> Dict: + """ + hand-crafted artisanal markdown can be provided in a .readme-partials.yml. + The following fields are currently supported: + + body: custom body to include in the usage section of the document. + samples_body: an optional body to place below the table of contents + in samples/README.md. + introduction: a more thorough introduction than metadata["description"]. + title: provide markdown to use as a custom title. + deprecation_warning: a warning to indicate that the library has been + deprecated and a pointer to an alternate option + """ + result: Dict[str, Dict] = {} + cwd_path = Path(os.getcwd()) + for file in files + _DEFAULT_PARTIAL_FILES: + partials_file = cwd_path / file + if os.path.exists(partials_file): + with open(partials_file) as f: + result.update(yaml.load(f, Loader=yaml.SafeLoader)) + return result diff --git a/hermetic_build/library_generation/owlbot/synthtool/gcp/samples.py b/hermetic_build/library_generation/owlbot/synthtool/gcp/samples.py new file mode 100644 index 0000000000..760fc06574 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/synthtool/gcp/samples.py @@ -0,0 +1,91 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import logging +import re +import os +import yaml +from typing import List, Dict + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +def _read_sample_metadata_comment(sample_file: str) -> Dict: + """Additional meta-information can be provided through embedded comments: + + // sample-metadata: + // title: ACL (Access Control) + // description: Demonstrates setting access control rules. + // usage: node iam.js --help + """ + sample_metadata = {} # type: Dict[str, str] + with open(sample_file) as f: + contents = f.read() + match = re.search( + r"(?P// *sample-metadata:([^\n]+|\n//)+)", contents, re.DOTALL + ) + if match: + # the metadata yaml is stored in a comments, remove the + # prefix so that we can parse the yaml contained. + sample_metadata_string = re.sub(r"((#|//) ?)", "", match.group("metadata")) + try: + sample_metadata = yaml.load( + sample_metadata_string, Loader=yaml.SafeLoader + )["sample-metadata"] + except yaml.scanner.ScannerError: + # warn and continue on bad metadata + logger.warning(f"bad metadata detected in {sample_file}") + return sample_metadata + + +def _sample_metadata(file: str) -> Dict[str, str]: + metadata = { + "title": _decamelize(os.path.splitext(os.path.basename(file))[0]), + "file": file, + } + return {**metadata, **_read_sample_metadata_comment(file)} + + +def all_samples(sample_globs: List[str]) -> List[Dict[str, str]]: + """Walks samples directory and builds up samples data-structure + + Args: + sample_globs: (List[str]): List of path globs to search for samples + + Returns: + A list of sample metadata in the format: + { + "title": "Requester Pays", + "file": "samples/requesterPays.js" + } + The file path is the relative path from the repository root. + """ + files = [] + for sample_glob in sample_globs: + for file in glob.glob(sample_glob, recursive=True): + files.append(file) + return [_sample_metadata(file) for file in sorted(files)] + + +def _decamelize(value: str): + """Parser to convert fooBar.js to Foo Bar.""" + if not value: + return "" + str_decamelize = re.sub("^.", value[0].upper(), value) # apple -> Apple. + str_decamelize = re.sub( + "([A-Z]+)([A-Z])([a-z0-9])", r"\1 \2\3", str_decamelize + ) # ACLBatman -> ACL Batman. + return re.sub("([a-z0-9])([A-Z])", r"\1 \2", str_decamelize) # FooBar -> Foo Bar. diff --git a/hermetic_build/library_generation/owlbot/synthtool/gcp/snippets.py b/hermetic_build/library_generation/owlbot/synthtool/gcp/snippets.py new file mode 100644 index 0000000000..5db00656b5 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/synthtool/gcp/snippets.py @@ -0,0 +1,124 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os +import re +from typing import Dict, List + +OPEN_SNIPPET_REGEX = r".*\[START ([a-z0-9_]+)\].*$" +CLOSE_SNIPPET_REGEX = r".*\[END ([a-z0-9_]+)\].*$" +OPEN_EXCLUDE_REGEX = r".*\[START_EXCLUDE\].*$" +CLOSE_EXCLUDE_REGEX = r".*\[END_EXCLUDE\].*$" + + +def _trim_leading_whitespace(lines: List[str]) -> List[str]: + """Trims leading, plain spaces from the snippet content. Finds the minimum + number of leading spaces, ignoring empty lines, and removes that number of + spaces from each line. + + Args: + lines (List[str]): Lines of content. These lines are newline terminated. + + Returns: + List of trimmed lines. + """ + + def number_of_leading_spaces(input: str) -> int: + return len(input) - len(input.lstrip(" ")) + + def is_empty_line(input: str) -> bool: + if re.match(r"^\s*$", input): + return True + return False + + leading_spaces = [ + number_of_leading_spaces(line) for line in lines if not is_empty_line(line) + ] + max_leading_spaces = min(leading_spaces) + return [ + "\n" if is_empty_line(line) else line[max_leading_spaces:] for line in lines + ] + + +def all_snippets_from_file(sample_file: str) -> Dict[str, str]: + """Reads in a sample file and parse out all contained snippets. + + Args: + sample_file (str): Sample file to parse. + + Returns: + Dictionary of snippet name to snippet code. + """ + if not os.path.exists(sample_file): + return {} + + snippet_lines = {} # type: Dict[str, List[str]] + open_snippets = set() + with open(sample_file) as f: + excluding = False + # Iterate over each line: + # - If the line matches an opening snippet tag, add that snippet tag to + # the set of open tags. + # - If the line matches a closing snippet tag, remove that snippet tag + # from the set of open tags. + # - If the line matches an opening exclude tag, record that we excluding + # content. + # - If the line matches a closing exclude tag, record that we are capturing + # content again. + # - Otherwise, if we are not excluding content, add the line to each of the + # open snippets + # + # This allows us to handle parsing nested or interleaved snippets and ignore + # blocks of code in the snippets + for line in f: + open_match = re.match(pattern=OPEN_SNIPPET_REGEX, string=line) + close_match = re.match(pattern=CLOSE_SNIPPET_REGEX, string=line) + open_exclude_match = re.match(pattern=OPEN_EXCLUDE_REGEX, string=line) + close_exclude_match = re.match(pattern=CLOSE_EXCLUDE_REGEX, string=line) + if open_match and not excluding: + open_snippets.add(open_match[1]) + if not open_match[1] in snippet_lines: + snippet_lines[open_match[1]] = [] + elif close_match and not excluding: + open_snippets.discard(close_match[1]) + elif open_exclude_match: + excluding = True + elif close_exclude_match: + excluding = False + elif not excluding: + for snippet in open_snippets: + snippet_lines[snippet].append(line) + + return { + snippet: "".join(_trim_leading_whitespace(lines)) + for snippet, lines in snippet_lines.items() + } + + +def all_snippets(snippet_globs: List[str]) -> Dict[str, str]: + """Walks the samples directory and parses snippets from each file. + + Args: + snippet_globs (List[str]): List of path globs to expand. + + Returns: + Dictionary of snippet name to snippet code. + """ + snippets = {} + for snippet_glob in snippet_globs: + for file in glob.glob(snippet_glob, recursive=True): + for snippet, code in all_snippets_from_file(file).items(): + snippets[snippet] = code + return snippets diff --git a/hermetic_build/library_generation/owlbot/synthtool/languages/java.py b/hermetic_build/library_generation/owlbot/synthtool/languages/java.py new file mode 100644 index 0000000000..3b8d321937 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/synthtool/languages/java.py @@ -0,0 +1,611 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import xml.etree.ElementTree as ET +import re +import requests +import yaml +import synthtool as s +import synthtool.gcp as gcp +import logging +from synthtool.gcp import common, samples, snippets +from pathlib import Path +from typing import Any, Optional, Dict, Iterable, List +from datetime import date + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +JAR_DOWNLOAD_URL = "https://github.com/google/google-java-format/releases/download/google-java-format-{version}/google-java-format-{version}-all-deps.jar" +DEFAULT_FORMAT_VERSION = "1.7" +CURRENT_YEAR = date.today().year +GOOD_LICENSE = f"""/* + * Copyright {CURRENT_YEAR} Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +""" +PROTOBUF_HEADER = "// Generated by the protocol buffer compiler. DO NOT EDIT!" +BAD_LICENSE = """/\\* + \\* Copyright \\d{4} Google LLC + \\* + \\* Licensed under the Apache License, Version 2.0 \\(the "License"\\); you may not use this file except + \\* in compliance with the License. You may obtain a copy of the License at + \\* + \\* http://www.apache.org/licenses/LICENSE-2.0 + \\* + \\* Unless required by applicable law or agreed to in writing, software distributed under the License + \\* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + \\* or implied. See the License for the specific language governing permissions and limitations under + \\* the License. + \\*/ +""" +DEFAULT_MIN_SUPPORTED_JAVA_VERSION = 8 +METADATA = "metadata" +LIBRARIES_BOM_VERSION = "libraries_bom_version" +LIBRARIES_BOM_VERSION_ENV_KEY = "SYNTHTOOL_LIBRARIES_BOM_VERSION" +HEADER_REGEX = re.compile("\\* Copyright \\d{4} Google LLC") +LIBRARY_VERSION = "library_version" +LIBRARY_VERSION_ENV_KEY = "SYNTHTOOL_LIBRARY_VERSION" + + +def _file_has_header(path: Path) -> bool: + """Return true if the file already contains a license header.""" + with open(path, "rt") as fp: + for line in fp: + if HEADER_REGEX.search(line): + return True + return False + + +def _filter_no_header(paths: Iterable[Path]) -> Iterable[Path]: + """Return a subset of files that do not already have a header.""" + for path in paths: + anchor = Path(path.anchor) + remainder = str(path.relative_to(path.anchor)) + for file in anchor.glob(remainder): + if not _file_has_header(file): + yield file + + +def fix_proto_headers(proto_root: Path) -> None: + """Helper to ensure that generated proto classes have appropriate license headers. + + If the file does not already contain a license header, inject one at the top of the file. + Some resource name classes may contain malformed license headers. In those cases, replace + those with our standard license header. + """ + s.replace( + _filter_no_header([proto_root / "src/**/*.java"]), + PROTOBUF_HEADER, + f"{GOOD_LICENSE}{PROTOBUF_HEADER}", + ) + # https://github.com/googleapis/gapic-generator/issues/3074 + s.replace( + [proto_root / "src/**/*Name.java", proto_root / "src/**/*Names.java"], + BAD_LICENSE, + GOOD_LICENSE, + ) + + +def fix_grpc_headers(grpc_root: Path, package_name: str = "unused") -> None: + """Helper to ensure that generated grpc stub classes have appropriate license headers. + + If the file does not already contain a license header, inject one at the top of the file. + """ + s.replace( + _filter_no_header([grpc_root / "src/**/*.java"]), + "^package (.*);", + f"{GOOD_LICENSE}package \\1;", + ) + + +def latest_maven_version(group_id: str, artifact_id: str) -> Optional[str]: + """Helper function to find the latest released version of a Maven artifact. + + Fetches metadata from Maven Central and parses out the latest released + version. + + Args: + group_id (str): The groupId of the Maven artifact + artifact_id (str): The artifactId of the Maven artifact + + Returns: + The latest version of the artifact as a string or None + """ + group_path = "/".join(group_id.split(".")) + url = ( + f"https://repo1.maven.org/maven2/{group_path}/{artifact_id}/maven-metadata.xml" + ) + response = requests.get(url) + if response.status_code >= 400: + return "0.0.0" + + return version_from_maven_metadata(response.text) + + +def version_from_maven_metadata(metadata: str) -> Optional[str]: + """Helper function to parse the latest released version from the Maven + metadata XML file. + + Args: + metadata (str): The XML contents of the Maven metadata file + + Returns: + The latest version of the artifact as a string or None + """ + root = ET.fromstring(metadata) + latest = root.find("./versioning/latest") + if latest is not None: + return latest.text + + return None + + +def _merge_release_please(destination_text: str): + handle_gh_release_key = "handleGHRelease" + branches_key = "branches" + config = yaml.safe_load(destination_text) + if handle_gh_release_key in config: + return destination_text + + config[handle_gh_release_key] = True + + if branches_key in config: + for branch in config[branches_key]: + branch[handle_gh_release_key] = True + return yaml.dump(config) + + +def _merge_common_templates( + source_text: str, destination_text: str, file_path: Path +) -> str: + # keep any existing pom.xml + if file_path.match("pom.xml") or file_path.match("sync-repo-settings.yaml"): + logger.debug(f"existing pom file found ({file_path}) - keeping the existing") + return destination_text + + if file_path.match("release-please.yml"): + return _merge_release_please(destination_text) + + # by default return the newly generated content + return source_text + + +def _common_template_metadata() -> Dict[str, Any]: + metadata = {} # type: Dict[str, Any] + repo_metadata = common._load_repo_metadata() + if repo_metadata: + metadata["repo"] = repo_metadata + group_id, artifact_id = repo_metadata["distribution_name"].split(":") + + metadata["latest_version"] = latest_maven_version( + group_id=group_id, artifact_id=artifact_id + ) + + metadata["latest_bom_version"] = latest_maven_version( + group_id="com.google.cloud", + artifact_id="libraries-bom", + ) + + metadata["samples"] = samples.all_samples(["samples/**/src/main/java/**/*.java"]) + metadata["snippets"] = snippets.all_snippets( + ["samples/**/src/main/java/**/*.java", "samples/**/pom.xml"] + ) + if repo_metadata and "min_java_version" in repo_metadata: + metadata["min_java_version"] = repo_metadata["min_java_version"] + else: + metadata["min_java_version"] = DEFAULT_MIN_SUPPORTED_JAVA_VERSION + + return metadata + + +def common_templates( + excludes: List[str] = [], + template_path: Optional[Path] = None, + **kwargs, +) -> None: + """Generate common templates for a Java Library + + Fetches information about the repository from the .repo-metadata.json file, + information about the latest artifact versions and copies the files into + their expected location. + + Args: + :param excludes: List of template paths to ignore + :param template_path: + :param kwargs: Additional options for CommonTemplates.java_library() + """ + metadata = _common_template_metadata() + kwargs[METADATA] = metadata + + # Generate flat to tell this repository is a split repo that have migrated + # to monorepo. The owlbot.py in the monorepo sets monorepo=True. + monorepo = kwargs.get("monorepo", False) + kwargs["monorepo"] = monorepo + split_repo = not monorepo + repo_metadata = metadata["repo"] + repo_short = repo_metadata["repo_short"] + if os.getenv(LIBRARIES_BOM_VERSION_ENV_KEY, default=None) is not None: + kwargs[METADATA][LIBRARIES_BOM_VERSION] = os.getenv( + LIBRARIES_BOM_VERSION_ENV_KEY + ) + kwargs[METADATA][LIBRARY_VERSION] = os.getenv(LIBRARY_VERSION_ENV_KEY) + # Special libraries that are not GAPIC_AUTO but in the monorepo + special_libs_in_monorepo = [ + "java-translate", + "java-dns", + "java-notification", + "java-resourcemanager", + ] + kwargs["migrated_split_repo"] = split_repo and ( + repo_metadata["library_type"] == "GAPIC_AUTO" + or (repo_short and repo_short in special_libs_in_monorepo) + ) + logger.info( + "monorepo: {}, split_repo: {}, library_type: {}," + " repo_short: {}, migrated_split_repo: {}".format( + monorepo, + split_repo, + repo_metadata["library_type"], + repo_short, + kwargs["migrated_split_repo"], + ) + ) + + templates = gcp.common.CommonTemplates(template_path=template_path).java_library( + **kwargs + ) + + s.copy([templates], excludes=excludes, merge=_merge_common_templates) + + +def custom_templates(files: List[str], **kwargs) -> None: + """Generate custom template files + + Fetches information about the repository from the .repo-metadata.json file, + information about the latest artifact versions and copies the files into + their expected location. + + Args: + files (List[str], optional): List of template paths to include + **kwargs: Additional options for CommonTemplates.render() + """ + kwargs["metadata"] = _common_template_metadata() + kwargs["metadata"]["partials"] = common.load_partials() + for file in files: + template = gcp.CommonTemplates().render(file, **kwargs) + s.copy([template]) + + +def remove_method(filename: str, signature: str): + """Helper to remove an entire method. + + Goes line-by-line to detect the start of the block. Determines + the end of the block by a closing brace at the same indentation + level. This requires the file to be correctly formatted. + + Example: consider the following class: + + class Example { + public void main(String[] args) { + System.out.println("Hello World"); + } + + public String foo() { + return "bar"; + } + } + + To remove the `main` method above, use: + + remove_method('path/to/file', 'public void main(String[] args)') + + Args: + filename (str): Path to source file + signature (str): Full signature of the method to remove. Example: + `public void main(String[] args)`. + """ + lines = [] + leading_regex = None + with open(filename, "r") as fp: + line = fp.readline() + while line: + # for each line, try to find the matching + regex = re.compile("(\\s*)" + re.escape(signature) + ".*") + match = regex.match(line) + if match: + leading_regex = re.compile(match.group(1) + "}") + line = fp.readline() + continue + + # not in a ignore block - preserve the line + if not leading_regex: + lines.append(line) + line = fp.readline() + continue + + # detect the closing tag based on the leading spaces + match = leading_regex.match(line) + if match: + # block is closed, resume capturing content + leading_regex = None + + line = fp.readline() + + with open(filename, "w") as fp: + for line in lines: + # print(line) + fp.write(line) + + +def copy_and_rename_method(filename: str, signature: str, before: str, after: str): + """Helper to make a copy an entire method and rename it. + + Goes line-by-line to detect the start of the block. Determines + the end of the block by a closing brace at the same indentation + level. This requires the file to be correctly formatted. + The method is copied over and renamed in the method signature. + The calls to both methods are separate and unaffected. + + Example: consider the following class: + + class Example { + public void main(String[] args) { + System.out.println("Hello World"); + } + + public String foo() { + return "bar"; + } + } + + To copy and rename the `main` method above, use: + + copy_and_rename_method('path/to/file', 'public void main(String[] args)', + 'main', 'foo1') + + Args: + filename (str): Path to source file + signature (str): Full signature of the method to remove. Example: + `public void main(String[] args)`. + before (str): name of the method to be copied + after (str): new name of the copied method + """ + lines = [] + method = [] + leading_regex = None + with open(filename, "r") as fp: + line = fp.readline() + while line: + # for each line, try to find the matching + regex = re.compile("(\\s*)" + re.escape(signature) + ".*") + match = regex.match(line) + if match: + leading_regex = re.compile(match.group(1) + "}") + lines.append(line) + method.append(line.replace(before, after)) + line = fp.readline() + continue + + lines.append(line) + # not in a ignore block - preserve the line + if leading_regex: + method.append(line) + else: + line = fp.readline() + continue + + # detect the closing tag based on the leading spaces + match = leading_regex.match(line) + if match: + # block is closed, resume capturing content + leading_regex = None + lines.append("\n") + lines.extend(method) + + line = fp.readline() + + with open(filename, "w") as fp: + for line in lines: + # print(line) + fp.write(line) + + +def add_javadoc(filename: str, signature: str, javadoc_type: str, content: List[str]): + """Helper to add a javadoc annoatation to a method. + + Goes line-by-line to detect the start of the block. + Then finds the existing method comment (if it exists). If the + comment already exists, it will append the javadoc annotation + to the javadoc block. Otherwise, it will create a new javadoc + comment block. + + Example: consider the following class: + + class Example { + public void main(String[] args) { + System.out.println("Hello World"); + } + + public String foo() { + return "bar"; + } + } + + To add a javadoc annotation the `main` method above, use: + + add_javadoc('path/to/file', 'public void main(String[] args)', + 'deprecated', 'Please use foo instead.') + + Args: + filename (str): Path to source file + signature (str): Full signature of the method to remove. Example: + `public void main(String[] args)`. + javadoc_type (str): The type of javadoc annotation. Example: `deprecated`. + content (List[str]): The javadoc lines + """ + lines: List[str] = [] + annotations: List[str] = [] + with open(filename, "r") as fp: + line = fp.readline() + while line: + # for each line, try to find the matching + regex = re.compile("(\\s*)" + re.escape(signature) + ".*") + match = regex.match(line) + if match: + leading_spaces = len(line) - len(line.lstrip()) + indent = leading_spaces * " " + last_line = lines.pop() + while last_line.lstrip() and last_line.lstrip()[0] == "@": + annotations.append(last_line) + last_line = lines.pop() + if last_line.strip() == "*/": + first = True + for content_line in content: + if first: + lines.append( + indent + + " * @" + + javadoc_type + + " " + + content_line + + "\n" + ) + first = False + else: + lines.append(indent + " * " + content_line + "\n") + lines.append(last_line) + else: + lines.append(last_line) + lines.append(indent + "/**\n") + first = True + for content_line in content: + if first: + lines.append( + indent + + " * @" + + javadoc_type + + " " + + content_line + + "\n" + ) + first = False + else: + lines.append(indent + " * " + content_line + "\n") + lines.append(indent + " */\n") + lines.extend(annotations[::-1]) + lines.append(line) + line = fp.readline() + + with open(filename, "w") as fp: + for line in lines: + # print(line) + fp.write(line) + + +def annotate_method(filename: str, signature: str, annotation: str): + """Helper to add an annotation to a method. + + Goes line-by-line to detect the start of the block. + Then adds the annotation above the found method signature. + + Example: consider the following class: + + class Example { + public void main(String[] args) { + System.out.println("Hello World"); + } + + public String foo() { + return "bar"; + } + } + + To add an annotation the `main` method above, use: + + annotate_method('path/to/file', 'public void main(String[] args)', + '@Generated()') + + Args: + filename (str): Path to source file + signature (str): Full signature of the method to remove. Example: + `public void main(String[] args)`. + annotation (str): Full annotation. Example: `@Deprecated` + """ + lines: List[str] = [] + with open(filename, "r") as fp: + line = fp.readline() + while line: + # for each line, try to find the matching + regex = re.compile("(\\s*)" + re.escape(signature) + ".*") + match = regex.match(line) + if match: + leading_spaces = len(line) - len(line.lstrip()) + indent = leading_spaces * " " + lines.append(indent + annotation + "\n") + lines.append(line) + line = fp.readline() + + with open(filename, "w") as fp: + for line in lines: + # print(line) + fp.write(line) + + +def deprecate_method(filename: str, signature: str, alternative: str): + """Helper to deprecate a method. + + Goes line-by-line to detect the start of the block. + Then adds the deprecation comment before the method signature. + The @Deprecation annotation is also added. + + Example: consider the following class: + + class Example { + public void main(String[] args) { + System.out.println("Hello World"); + } + + public String foo() { + return "bar"; + } + } + + To deprecate the `main` method above, use: + + deprecate_method('path/to/file', 'public void main(String[] args)', + DEPRECATION_WARNING.format(new_method="foo")) + + Args: + filename (str): Path to source file + signature (str): Full signature of the method to remove. Example: + `public void main(String[] args)`. + alternative: DEPRECATION WARNING: multiline javadoc comment with user + specified leading open/close comment tags + """ + add_javadoc(filename, signature, "deprecated", alternative.splitlines()) + annotate_method(filename, signature, "@Deprecated") diff --git a/hermetic_build/library_generation/owlbot/synthtool/sources/templates.py b/hermetic_build/library_generation/owlbot/synthtool/sources/templates.py new file mode 100644 index 0000000000..7db2ea6a92 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/synthtool/sources/templates.py @@ -0,0 +1,79 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Union, List +from pathlib import Path +import jinja2 +import tempfile + + +PathOrStr = Union[str, Path] + + +def _make_env(location): + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(str(location)), + autoescape=False, + keep_trailing_newline=True, + ) + return env + + +def _render_to_path(env, template_name, dest, params): + template = env.get_template(template_name) + + output = template.stream(**params) + + if template_name.endswith(".j2"): + template_name = template.name[:-3] + + dest = dest / template_name + dest.parent.mkdir(parents=True, exist_ok=True) + + with dest.open("w") as fh: + output.dump(fh) + + # Copy file mode over + source_path = Path(template.filename) + mode = source_path.stat().st_mode + dest.chmod(mode) + + return dest + + +class Templates: + def __init__(self, location: PathOrStr) -> None: + self.env = _make_env(location) + self.source_path = Path(location) + self.dir = Path(tempfile.mkdtemp()) + + def render(self, template_name: str, subdir: PathOrStr = ".", **kwargs) -> Path: + return _render_to_path(self.env, template_name, self.dir / subdir, kwargs) + + +class TemplateGroup: + def __init__(self, location: PathOrStr, excludes: List[str] = []) -> None: + self.env = _make_env(location) + self.dir = Path(tempfile.mkdtemp()) + self.excludes = excludes + + def render(self, subdir: PathOrStr = ".", **kwargs) -> Path: + for template_name in self.env.list_templates(): + if template_name not in self.excludes: + print(template_name) + _render_to_path(self.env, template_name, self.dir / subdir, kwargs) + else: + print(f"Skipping: {template_name}") + + return self.dir diff --git a/hermetic_build/library_generation/owlbot/synthtool/transforms.py b/hermetic_build/library_generation/owlbot/synthtool/transforms.py new file mode 100644 index 0000000000..cf87995847 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/synthtool/transforms.py @@ -0,0 +1,314 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import shutil +from typing import Callable, Iterable, Union, List, Optional +import os +import re +import sys +import logging + +from synthtool import _tracked_paths + +PathOrStr = Union[str, Path] +ListOfPathsOrStrs = Iterable[Union[str, Path]] + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + + +class MissingSourceError(Exception): + pass + + +def _expand_paths(paths: ListOfPathsOrStrs, root: PathOrStr = None) -> Iterable[Path]: + """Given a list of globs/paths, expands them into a flat sequence, + expanding globs as necessary.""" + if paths is None: + return [] + + if isinstance(paths, (str, Path)): + paths = [paths] + + if root is None: + root = Path(".") + + # ensure root is a path + root = Path(root) + + # record name of synth script so we don't try to do transforms on it + synth_script_name = sys.argv[0] + + for path in paths: + if isinstance(path, Path): + if path.is_absolute(): + anchor = Path(path.anchor) + remainder = str(path.relative_to(path.anchor)) + yield from anchor.glob(remainder) + else: + yield from root.glob(str(path)) + else: + yield from ( + p + for p in root.glob(path) + if p.absolute() != Path(synth_script_name).absolute() + ) + + +def _filter_files(paths: Iterable[Path]) -> Iterable[Path]: + """Returns only the paths that are files (no directories).""" + + return (path for path in paths if path.is_file() and os.access(path, os.W_OK)) + + +def _merge_file( + source_path: Path, dest_path: Path, merge: Callable[[str, str, Path], str] +): + """ + Writes to the destination the result of merging the source with the + existing destination contents, using the given merge function. + + The merge function must take three arguments: the source contents, the + old destination contents, and a Path to the file to be written. + """ + + with source_path.open("r") as source_file: + source_text = source_file.read() + + with dest_path.open("r+") as dest_file: + dest_text = dest_file.read() + + final_text = merge(source_text, dest_text, dest_path) + + # use the source file's file permission mode + os.chmod(dest_path, os.stat(source_path).st_mode) + if final_text != dest_text: + dest_file.seek(0) + dest_file.write(final_text) + dest_file.truncate() + else: + dest_path.touch() + + +def _copy_dir_to_existing_dir( + source: Path, + destination: Path, + excludes: ListOfPathsOrStrs = None, + merge: Callable[[str, str, Path], str] = None, +) -> bool: + """ + copies files over existing files to an existing directory + this function does not copy empty directories. + + Returns: True if any files were copied, False otherwise. + """ + copied = False + + if not excludes: + excludes = [] + for root, _, files in os.walk(source): + for name in files: + rel_path = str(Path(root).relative_to(source)) + dest_dir = destination / rel_path + dest_path = dest_dir / name + exclude = [ + e + for e in excludes + if ( + Path(e) == _tracked_paths.relativize(root) + or Path(e) == _tracked_paths.relativize(Path(root) / name) + ) + ] + if not exclude: + os.makedirs(str(dest_dir), exist_ok=True) + source_path = Path(os.path.join(root, name)) + if merge is not None and dest_path.is_file(): + try: + _merge_file(source_path, dest_path, merge) + except Exception: + logger.exception( + "_merge_file failed for %s, fall back to copy", + source_path, + ) + shutil.copy2(str(source_path), str(dest_path)) + else: + shutil.copy2(str(source_path), str(dest_path)) + copied = True + + return copied + + +def move( + sources: ListOfPathsOrStrs, + destination: PathOrStr = None, + excludes: ListOfPathsOrStrs = None, + merge: Callable[[str, str, Path], str] = None, + required: bool = False, +) -> bool: + """ + copy file(s) at source to current directory, preserving file mode. + + Args: + sources (ListOfPathsOrStrs): Glob pattern(s) to copy + destination (PathOrStr): Destination folder for copied files + excludes (ListOfPathsOrStrs): Glob pattern(s) of files to skip + merge (Callable[[str, str, Path], str]): Callback function for merging files + if there is an existing file. + required (bool): If required and no source files are copied, throws a MissingSourceError + + Returns: + True if any files were copied, False otherwise. + """ + copied = False + + for source in _expand_paths(sources): + if destination is None: + canonical_destination = _tracked_paths.relativize(source) + else: + canonical_destination = Path(destination) + + if excludes: + excludes = [ + _tracked_paths.relativize(e) for e in _expand_paths(excludes, source) + ] + else: + excludes = [] + if source.is_dir(): + copied = copied or _copy_dir_to_existing_dir( + source, canonical_destination, excludes=excludes, merge=merge + ) + elif source not in excludes: + # copy individual file + if merge is not None and canonical_destination.is_file(): + try: + _merge_file(source, canonical_destination, merge) + except Exception: + logger.exception( + "_merge_file failed for %s, fall back to copy", source + ) + shutil.copy2(source, canonical_destination) + else: + shutil.copy2(source, canonical_destination) + copied = True + + if not copied: + if required: + raise MissingSourceError( + f"No files in sources {sources} were copied. Does the source " + f"contain files?" + ) + else: + logger.warning( + f"No files in sources {sources} were copied. Does the source " + f"contain files?" + ) + + return copied + + +def _replace_in_file(path, expr, replacement): + try: + with path.open("r+") as fh: + return _replace_in_file_handle(fh, expr, replacement) + except UnicodeDecodeError: + pass # It's a binary file. Try again with a binary regular expression. + flags = expr.flags & ~re.UNICODE + expr = re.compile(expr.pattern.encode(), flags) + with path.open("rb+") as fh: + return _replace_in_file_handle(fh, expr, replacement.encode()) + + +def _replace_in_file_handle(fh, expr, replacement): + content = fh.read() + content, count = expr.subn(replacement, content) + + # Don't bother writing the file if we didn't change + # anything. + if count: + fh.seek(0) + fh.write(content) + fh.truncate() + return count + + +def replace( + sources: ListOfPathsOrStrs, before: str, after: str, flags: int = re.MULTILINE +) -> int: + """Replaces occurrences of before with after in all the given sources. + + Returns: + The number of times the text was found and replaced across all files. + """ + expr = re.compile(before, flags=flags or 0) + paths = _filter_files(_expand_paths(sources, ".")) + + if not paths: + logger.warning(f"No files were found in sources {sources} for replace()") + + count_replaced = 0 + for path in paths: + replaced = _replace_in_file(path, expr, after) + count_replaced += replaced + if replaced: + logger.info(f"Replaced {before!r} in {path}.") + + if not count_replaced: + logger.warning( + f"No replacements made in {sources} for pattern {before}, maybe " + "replacement is no longer needed?" + ) + return count_replaced + + +def get_staging_dirs( + default_version: Optional[str] = None, staging_path: Optional[str] = None +) -> List[Path]: + """Returns the list of directories, one per version, copied from + https://github.com/googleapis/googleapis-gen. Will return in lexical sorting + order with the exception of the default_version which will be last (if specified). + + Args: + default_version: the default version of the API. The directory for this version + will be the last item in the returned list if specified. + staging_path: the path to the staging directory. + + Returns: the empty list if no file were copied. + """ + + if staging_path: + staging = Path(staging_path) + else: + staging = Path("owl-bot-staging") + if staging.is_dir(): + # Collect the subdirectories of the staging directory. + versions = [v.name for v in staging.iterdir() if v.is_dir()] + # Reorder the versions so the default version always comes last. + versions = [v for v in versions if v != default_version] + versions.sort() + if default_version is not None: + versions += [default_version] + dirs = [staging / v for v in versions] + for dir in dirs: + _tracked_paths.add(dir) + return dirs + else: + return [] + + +def remove_staging_dirs(): + """Removes all the staging directories.""" + staging = Path("owl-bot-staging") + if staging.is_dir(): + shutil.rmtree(staging) diff --git a/hermetic_build/library_generation/owlbot/templates/clirr/clirr-ignored-differences.xml.j2 b/hermetic_build/library_generation/owlbot/templates/clirr/clirr-ignored-differences.xml.j2 new file mode 100644 index 0000000000..3fc907802f --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/clirr/clirr-ignored-differences.xml.j2 @@ -0,0 +1,80 @@ + + + +{% for proto_path in proto_paths %} + 7012 + {{proto_path}}/*OrBuilder + * get*(*) + + + 7012 + {{proto_path}}/*OrBuilder + boolean contains*(*) + + + 7012 + {{proto_path}}/*OrBuilder + boolean has*(*) + + + + 7006 + {{proto_path}}/** + * getDefaultInstanceForType() + ** + + + 7006 + {{proto_path}}/** + * addRepeatedField(*) + ** + + + 7006 + {{proto_path}}/** + * clear() + ** + + + 7006 + {{proto_path}}/** + * clearField(*) + ** + + + 7006 + {{proto_path}}/** + * clearOneof(*) + ** + + + 7006 + {{proto_path}}/** + * clone() + ** + + + 7006 + {{proto_path}}/** + * mergeUnknownFields(*) + ** + + + 7006 + {{proto_path}}/** + * setField(*) + ** + + + 7006 + {{proto_path}}/** + * setRepeatedField(*) + ** + + + 7006 + {{proto_path}}/** + * setUnknownFields(*) + ** + {% endfor %} + diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/CODEOWNERS b/hermetic_build/library_generation/owlbot/templates/java_library/.github/CODEOWNERS new file mode 100644 index 0000000000..5002a1b08f --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/CODEOWNERS @@ -0,0 +1,20 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. + +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax +{% if 'codeowner_team' in metadata['repo'] %} +# The {{ metadata['repo']['codeowner_team'] }} is the default owner for changes in this repo +* @googleapis/yoshi-java {{ metadata['repo']['codeowner_team'] }} +{% if 'library_type' in metadata['repo'] and metadata['repo']['library_type'] != 'GAPIC_AUTO' %} +# for handwritten libraries, keep codeowner_team in .repo-metadata.json as owner +**/*.java {{ metadata['repo']['codeowner_team'] }} +{% endif %} +{% else %} +* @googleapis/yoshi-java +{% endif %} +# The java-samples-reviewers team is the default owner for samples changes +samples/**/*.java @googleapis/java-samples-reviewers + +# Generated snippets should not be owned by samples reviewers +samples/snippets/generated/ @googleapis/yoshi-java diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/bug_report.md b/hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..c7539a6878 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,56 @@ +--- +name: Bug report +about: Create a report to help us improve + +--- +{% if migrated_split_repo %} +:bus: This library has moved to +[google-cloud-java/{{ metadata['repo']['repo_short'] }}]( +https://github.com/googleapis/google-cloud-java/tree/main/{{ metadata['repo']['repo_short'] }}). +This repository will be archived in the future. +{% endif %} +Thanks for stopping by to let us know something could be better! + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. + +Please run down the following list and make sure you've tried the usual "quick fixes": + + - Search the issues already opened: https://github.com/googleapis/{{metadata['repo']['repo_short']}}/issues + - Check for answers on StackOverflow: http://stackoverflow.com/questions/tagged/google-cloud-platform + +If you are still having issues, please include as much information as possible: + +#### Environment details + +1. Specify the API at the beginning of the title. For example, "BigQuery: ..."). + General, Core, and Other are also allowed as types +2. OS type and version: +3. Java version: +4. {{metadata['repo']['name']}} version(s): + +#### Steps to reproduce + + 1. ? + 2. ? + +#### Code example + +```java +// example +``` + +#### Stack trace +``` +Any relevant stacktrace here. +``` + +#### External references such as API reference guides + +- ? + +#### Any additional information below + + +Following these steps guarantees the quickest resolution possible. + +Thanks! diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/feature_request.md b/hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..f89a7dc59e --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,26 @@ +--- +name: Feature request +about: Suggest an idea for this library + +--- +{% if migrated_split_repo %} +:bus: This library has moved to +[google-cloud-java/{{ metadata['repo']['repo_short'] }}]( +https://github.com/googleapis/google-cloud-java/tree/main/{{ metadata['repo']['repo_short'] }}). +This repository will be archived in the future. +{% endif %} +Thanks for stopping by to let us know something could be better! + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. + +**Is your feature request related to a problem? Please describe.** +What the problem is. Example: I'm always frustrated when [...] + +**Describe the solution you'd like** +What you want to happen. + +**Describe alternatives you've considered** +Any alternative solutions or features you've considered. + +**Additional context** +Any other context or screenshots about the feature request. diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/support_request.md b/hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/support_request.md new file mode 100644 index 0000000000..9958690321 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/ISSUE_TEMPLATE/support_request.md @@ -0,0 +1,7 @@ +--- +name: Support request +about: If you have a support contract with Google, please create an issue in the Google Cloud Support console. + +--- + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/PULL_REQUEST_TEMPLATE.md b/hermetic_build/library_generation/owlbot/templates/java_library/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..b3640828ab --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,10 @@ +Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: +- [ ] Make sure to open an issue as a [bug/issue](https://github.com/{{ metadata['repo']['repo'] }}/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea +- [ ] Ensure the tests and linter pass +- [ ] Code coverage does not decrease (if any source code was changed) +- [ ] Appropriate docs were updated (if necessary) + +Fixes # ☕️ + +If you write sample code, please follow the [samples format]( +https://github.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md). diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/auto-label.yaml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/auto-label.yaml new file mode 100644 index 0000000000..4caef688b7 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/auto-label.yaml @@ -0,0 +1,15 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +requestsize: + enabled: true diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/blunderbuss.yml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/blunderbuss.yml new file mode 100644 index 0000000000..2176b05432 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/blunderbuss.yml @@ -0,0 +1,7 @@ +# Configuration for the Blunderbuss GitHub app. For more info see +# https://github.com/googleapis/repo-automation-bots/tree/main/packages/blunderbuss +assign_prs_by: +- labels: + - samples + to: + - googleapis/java-samples-reviewers \ No newline at end of file diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/dependabot.yml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/dependabot.yml new file mode 100644 index 0000000000..203f9eaccf --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/dependabot.yml @@ -0,0 +1,19 @@ +version: 2 +updates: + - package-ecosystem: "maven" + directory: "/" + schedule: + interval: "daily" + # Disable version updates for Maven dependencies + # we use renovate-bot as well as shared-dependencies BOM to update maven dependencies. + ignore: + - dependency-name: "*" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + # Disable version updates for pip dependencies + # If a security vulnerability comes in, we will be notified about + # it via template in the synthtool repository. + ignore: + - dependency-name: "*" diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/release-please.yml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/release-please.yml new file mode 100644 index 0000000000..8ca7f9cabc --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/release-please.yml @@ -0,0 +1,3 @@ +bumpMinorPreMajor: true +handleGHRelease: true +releaseType: java-yoshi diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/release-trigger.yml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/release-trigger.yml new file mode 100644 index 0000000000..5056d3a13b --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/release-trigger.yml @@ -0,0 +1,2 @@ +enabled: true +multiScmName: {{ metadata['repo']['repo_short'] }} diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/scripts/update_generation_config.sh b/hermetic_build/library_generation/owlbot/templates/java_library/.github/scripts/update_generation_config.sh new file mode 100644 index 0000000000..561a313040 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/scripts/update_generation_config.sh @@ -0,0 +1,121 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to update googleapis_commitish, gapic_generator_version, +# and libraries_bom_version in generation configuration at the time of running +# and create a pull request. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. jq + +# Utility functions +# Get the latest released version of a Maven artifact. +function get_latest_released_version() { + local group_id=$1 + local artifact_id=$2 + latest=$(curl -s "https://search.maven.org/solrsearch/select?q=g:${group_id}+AND+a:${artifact_id}&core=gav&rows=500&wt=json" | jq -r '.response.docs[] | select(.v | test("^[0-9]+(\\.[0-9]+)*$")) | .v' | sort -V | tail -n 1) + echo "${latest}" +} + +# Update a key to a new value in the generation config. +function update_config() { + local key_word=$1 + local new_value=$2 + local file=$3 + echo "Update ${key_word} to ${new_value} in ${file}" + sed -i -e "s/^${key_word}.*$/${key_word}: ${new_value}/" "${file}" +} + +# The parameters of this script is: +# 1. base_branch, the base branch of the result pull request. +# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --base_branch) + base_branch="$2" + shift + ;; + --repo) + repo="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${base_branch}" ]; then + echo "missing required argument --base_branch" + exit 1 +fi + +if [ -z "${repo}" ]; then + echo "missing required argument --repo" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config="generation_config.yaml" + echo "Use default generation config: ${generation_config}" +fi + +current_branch="generate-libraries-${base_branch}" +title="chore: Update generation configuration at $(date)" + +# try to find a open pull request associated with the branch +pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") +# create a branch if there's no open pull request associated with the +# branch; otherwise checkout the pull request. +if [ -z "${pr_num}" ]; then + git checkout -b "${current_branch}" +else + gh pr checkout "${pr_num}" +fi + +mkdir tmp-googleapis +# use partial clone because only commit history is needed. +git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis +pushd tmp-googleapis +git pull +latest_commit=$(git rev-parse HEAD) +popd +rm -rf tmp-googleapis +update_config "googleapis_commitish" "${latest_commit}" "${generation_config}" + +# update gapic-generator-java version to the latest +latest_version=$(get_latest_released_version "com.google.api" "gapic-generator-java") +update_config "gapic_generator_version" "${latest_version}" "${generation_config}" + +# update libraries-bom version to the latest +latest_version=$(get_latest_released_version "com.google.cloud" "libraries-bom") +update_config "libraries_bom_version" "${latest_version}" "${generation_config}" + +git add "${generation_config}" +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "The latest generation config is not changed." + echo "Skip committing to the pull request." + exit 0 +fi +git commit -m "${title}" +if [ -z "${pr_num}" ]; then + git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" + git fetch -q --unshallow remote_repo + git push -f remote_repo "${current_branch}" + gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" +else + git push + gh pr edit "${pr_num}" --title "${title}" --body "${title}" +fi diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/snippet-bot.yml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/snippet-bot.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml new file mode 100644 index 0000000000..bbfd4c0314 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/sync-repo-settings.yaml @@ -0,0 +1,64 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Whether or not rebase-merging is enabled on this repository. +# Defaults to `true` +rebaseMergeAllowed: false + +# Whether or not squash-merging is enabled on this repository. +# Defaults to `true` +squashMergeAllowed: true + +# Whether or not PRs are merged with a merge commit on this repository. +# Defaults to `false` +mergeCommitAllowed: false + +# Rules for main branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `main` +- pattern: main + # Can admins overwrite branch protection. + # Defaults to `true` + isAdminEnforced: true + # Number of approving reviews required to update matching branches. + # Defaults to `1` + requiredApprovingReviewCount: 1 + # Are reviews from code owners required to update matching branches. + # Defaults to `false` + requiresCodeOwnerReviews: true + # Require up to date branches + requiresStrictStatusChecks: false + # List of required status check contexts that must pass for commits to be accepted to matching branches. + requiredStatusCheckContexts: + - "dependencies (17)" + - "lint" + - "javadoc" + - "units (8)" + - "units (11)" + - "Kokoro - Test: Integration" + - "cla/google" + - "OwlBot Post Processor" + - "Kokoro - Test: Java GraalVM Native Image" + - "Kokoro - Test: Java 17 GraalVM Native Image" +# List of explicit permissions to add (additive only) +permissionRules: +- team: yoshi-admins + permission: admin +- team: yoshi-java-admins + permission: admin +- team: yoshi-java + permission: push +- team: java-samples-reviewers + permission: push + diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/trusted-contribution.yml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/trusted-contribution.yml new file mode 100644 index 0000000000..88d3ac9bf1 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/trusted-contribution.yml @@ -0,0 +1,9 @@ +trustedContributors: +- renovate-bot +- gcf-owl-bot[bot] + +annotations: +- type: comment + text: "/gcbrun" +- type: label + text: "kokoro:force-run" diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/approve-readme.yaml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/approve-readme.yaml new file mode 100644 index 0000000000..bbef6d37cc --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/approve-readme.yaml @@ -0,0 +1,69 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Github action job to test core java library features on +# downstream client libraries before they are released. +on: + pull_request: +name: auto-merge-readme +jobs: + approve: + runs-on: ubuntu-latest + if: github.repository_owner == 'googleapis' && github.head_ref == 'autosynth-readme' + steps: + - uses: actions/github-script@v7 + with: + github-token: {{ '${{secrets.YOSHI_APPROVER_TOKEN}}' }} + script: | + // only approve PRs from yoshi-automation + if (context.payload.pull_request.user.login !== "yoshi-automation") { + return; + } + + // only approve PRs like "chore: release " + if (!context.payload.pull_request.title === "chore: regenerate README") { + return; + } + + // only approve PRs with README.md and synth.metadata changes + const files = new Set( + ( + await github.paginate( + github.pulls.listFiles.endpoint({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + }) + ) + ).map(file => file.filename) + ); + if (files.size != 2 || !files.has("README.md") || !files.has(".github/readme/synth.metadata/synth.metadata")) { + return; + } + + // approve README regeneration PR + await github.pulls.createReview({ + owner: context.repo.owner, + repo: context.repo.repo, + body: 'Rubber stamped PR!', + pull_number: context.payload.pull_request.number, + event: 'APPROVE' + }); + + // attach automerge label + await github.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + labels: ['automerge'] + }); diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/ci.yaml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/ci.yaml new file mode 100644 index 0000000000..50487eeb3b --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/ci.yaml @@ -0,0 +1,123 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Github action job to test core java library features on +# downstream client libraries before they are released. +on: + push: + branches: + - main + pull_request: +name: ci +jobs: + units: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + java: [11, 17, 21] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{'{{matrix.java}}'}} + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: test + units-java8: + # Building using Java 17 and run the tests with Java 8 runtime + name: "units (8)" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: temurin + - name: "Set jvm system property environment variable for surefire plugin (unit tests)" + # Maven surefire plugin (unit tests) allows us to specify JVM to run the tests. + # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm + run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java" >> $GITHUB_ENV + shell: bash + - uses: actions/setup-java@v4 + with: + java-version: 17 + distribution: temurin + - run: .kokoro/build.sh + env: + JOB_TYPE: test + windows: + runs-on: windows-latest + steps: + - name: Support longpaths + run: git config --system core.longpaths true + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 8 + - run: java -version + - run: .kokoro/build.bat + env: + JOB_TYPE: test + dependencies: + runs-on: ubuntu-latest + strategy: + matrix: + java: [17] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{'{{matrix.java}}'}} + - run: java -version + - run: .kokoro/dependencies.sh + javadoc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: javadoc + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: lint + clirr: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 8 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: clirr diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml new file mode 100644 index 0000000000..7c5ec7865e --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/renovate_config_check.yaml @@ -0,0 +1,25 @@ +name: Renovate Bot Config Validation + +on: + pull_request: + paths: + - 'renovate.json' + +jobs: + renovate_bot_config_validation: + runs-on: ubuntu-22.04 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install Renovate and Config Validator + run: | + npm install -g npm@latest + npm install --global renovate + renovate-config-validator diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/samples.yaml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/samples.yaml new file mode 100644 index 0000000000..03b2939567 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/samples.yaml @@ -0,0 +1,30 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Github action job to test core java library features on +# downstream client libraries before they are released. +on: + pull_request: +name: samples +jobs: + checkstyle: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 8 + - name: Run checkstyle + run: mvn -P lint --quiet --batch-mode checkstyle:check + working-directory: samples/snippets diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml new file mode 100644 index 0000000000..0dbdc19a4a --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.github/workflows/update_generation_config.yaml @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Update generation configuration +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: +{% raw %} +jobs: + update-generation-config: + runs-on: ubuntu-22.04 + env: + # the branch into which the pull request is merged + base_branch: main + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Update params in generation config to latest + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash .github/scripts/update_generation_config.sh \ + --base_branch "${base_branch}" \ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} +{% endraw %} diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/build.sh b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/build.sh new file mode 100755 index 0000000000..eda70322e3 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +cd github/synthtool + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Run tests +nox -s lint test + +# remove all files, preventing kokoro from trying to sync them. +rm -rf * diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/common.cfg b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/common.cfg new file mode 100644 index 0000000000..653d0e51ec --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/common.cfg @@ -0,0 +1,19 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "synthtool/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python" +} + +# Tell the trampoline which build file to use. +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/synthtool/.kokoro/build.sh" +} diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/continuous.cfg b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/continuous.cfg new file mode 100644 index 0000000000..18a4c35325 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/continuous.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit.cfg b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit.cfg new file mode 100644 index 0000000000..18a4c35325 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/presubmit.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/trampoline.sh b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/trampoline.sh new file mode 100755 index 0000000000..0efc3be388 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/.kokoro/trampoline.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -eo pipefail +# Always run the cleanup script, regardless of the success of bouncing into +# the container. +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md b/hermetic_build/library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..2add2547a8 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md @@ -0,0 +1,94 @@ + +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of +experience, education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, +offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +This Code of Conduct also applies outside the project spaces when the Project +Steward has a reasonable belief that an individual's behavior may have a +negative impact on the project or its community. + +## Conflict Resolution + +We do not believe that all conflict is bad; healthy debate and disagreement +often yield positive results. However, it is never okay to be disrespectful or +to engage in behavior that violates the project’s code of conduct. + +If you see someone violating the code of conduct, you are encouraged to address +the behavior directly with those involved. Many issues can be resolved quickly +and easily, and this gives people more control over the outcome of their +dispute. If you are unable to resolve the matter for any reason, or if the +behavior is threatening or harassing, report it. We are dedicated to providing +an environment where participants feel welcome and safe. + +Reports should be directed to *googleapis-stewards@google.com*, the +Project Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to +receive and address reported violations of the code of conduct. They will then +work with a committee consisting of representatives from the Open Source +Programs Office and the Google Open Source Strategy team. If for any reason you +are uncomfortable reaching out to the Project Steward, please email +opensource@google.com. + +We will investigate every complaint, but you may not receive a direct response. +We will use our discretion in determining when and how to follow up on reported +incidents, which may range from not taking action to permanent expulsion from +the project and project-sponsored spaces. We will notify the accused of the +report and provide them an opportunity to discuss it before any action is taken. +The identity of the reporter will be omitted from the details of the report +supplied to the accused. In potentially harmful situations, such as ongoing +harassment or threats to anyone's safety, we may take action without notice. + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/CONTRIBUTING.md b/hermetic_build/library_generation/owlbot/templates/java_library/CONTRIBUTING.md new file mode 100644 index 0000000000..b65dd279c9 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/CONTRIBUTING.md @@ -0,0 +1,92 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows +[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). + +## Building the project + +To build, package, and run all unit tests run the command + +``` +mvn clean verify +``` + +### Running Integration tests + +To include integration tests when building the project, you need access to +a GCP Project with a valid service account. + +For instructions on how to generate a service account and corresponding +credentials JSON see: [Creating a Service Account][1]. + +Then run the following to build, package, run all unit tests and run all +integration tests. + +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account.json +mvn -Penable-integration-tests clean verify +``` + +## Code Samples + +All code samples must be in compliance with the [java sample formatting guide][3]. +Code Samples must be bundled in separate Maven modules. + +The samples must be separate from the primary project for a few reasons: +1. Primary projects have a minimum Java version of Java 8 whereas samples can have + Java version of Java 11. Due to this we need the ability to + selectively exclude samples from a build run. +2. Many code samples depend on external GCP services and need + credentials to access the service. +3. Code samples are not released as Maven artifacts and must be excluded from + release builds. + +### Building + +```bash +mvn clean verify +``` + +Some samples require access to GCP services and require a service account: + +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account.json +mvn clean verify +``` + +### Code Formatting + +Code in this repo is formatted with +[google-java-format](https://github.com/google/google-java-format). +To run formatting on your project, you can run: +``` +mvn com.coveo:fmt-maven-plugin:format +``` + +[1]: https://cloud.google.com/docs/authentication/getting-started#creating_a_service_account +[2]: https://maven.apache.org/settings.html#Active_Profiles +[3]: https://github.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md \ No newline at end of file diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/LICENSE b/hermetic_build/library_generation/owlbot/templates/java_library/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/README.md b/hermetic_build/library_generation/owlbot/templates/java_library/README.md new file mode 100644 index 0000000000..35b1b34492 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/README.md @@ -0,0 +1,288 @@ +{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} +{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} +{% set version = metadata['library_version'] -%} +{% set repo_short = metadata['repo']['repo'].split('/')|last -%} + +# Google {{ metadata['repo']['name_pretty'] }} Client for Java + +Java idiomatic client for [{{metadata['repo']['name_pretty']}}][product-docs]. + +[![Maven][maven-version-image]][maven-version-link] +![Stability][stability-image] + +- [Product Documentation][product-docs] +- [Client Library Documentation][javadocs] +{% if 'partials' in metadata and metadata['partials']['deprecation_warning'] -%} +{{ metadata['partials']['deprecation_warning'] }} +{% elif metadata['repo']['release_level'] in ['preview'] %} +> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. +{% endif %} +{% if migrated_split_repo %} +:bus: In October 2022, this library has moved to +[google-cloud-java/{{ metadata['repo']['repo_short'] }}]( +https://github.com/googleapis/google-cloud-java/tree/main/{{ metadata['repo']['repo_short'] }}). +This repository will be archived in the future. +Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). +The Maven artifact coordinates (`{{ group_id }}:{{ artifact_id }}`) remain the same. +{% endif %} +## Quickstart + +{% if 'snippets' in metadata and metadata['snippets'][metadata['repo']['api_shortname'] + '_install_with_bom'] -%} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml +{{ metadata['snippets'][metadata['repo']['api_shortname'] + '_install_with_bom'] }} +``` + +If you are using Maven without the BOM, add this to your dependencies: +{% elif monorepo %} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml + + + + com.google.cloud + libraries-bom + {{ metadata['libraries_bom_version'] }} + pom + import + + + + + + + {{ group_id }} + {{ artifact_id }} + + +``` + +If you are using Maven without the BOM, add this to your dependencies: +{% else %} +If you are using Maven, add this to your pom.xml file: +{% endif %} + +```xml +{% if 'snippets' in metadata and metadata['snippets'][metadata['repo']['api_shortname'] + '_install_without_bom'] -%} +{{ metadata['snippets'][metadata['repo']['api_shortname'] + '_install_without_bom'] }} +{% else -%} + + {{ group_id }} + {{ artifact_id }} + {{ version }} + +{% endif -%} +``` + +{% if 'snippets' in metadata and metadata['snippets'][metadata['repo']['api_shortname'] + '_install_with_bom'] -%} +If you are using Gradle 5.x or later, add this to your dependencies: + +```Groovy +implementation platform('com.google.cloud:libraries-bom:{{metadata['libraries_bom_version']}}') + +implementation '{{ group_id }}:{{ artifact_id }}' +``` +{% endif -%} + +If you are using Gradle without BOM, add this to your dependencies: + +```Groovy +implementation '{{ group_id }}:{{ artifact_id }}:{{ version }}' +``` + +If you are using SBT, add this to your dependencies: + +```Scala +libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ version }}" +``` + +## Authentication + +See the [Authentication][authentication] section in the base directory's README. + +## Authorization + +The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired {{metadata['repo']['name_pretty']}} APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the {{metadata['repo']['name_pretty']}} API calls. + +## Getting Started + +### Prerequisites + +You will need a [Google Cloud Platform Console][developer-console] project with the {{metadata['repo']['name_pretty']}} [API enabled][enable-api]. +{% if metadata['repo']['requires_billing'] %}You will need to [enable billing][enable-billing] to use Google {{metadata['repo']['name_pretty']}}.{% endif %} +[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by +[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: +`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +### Installation and setup + +You'll need to obtain the `{{ artifact_id }}` library. See the [Quickstart](#quickstart) section +to add `{{ artifact_id }}` as a dependency in your code. + +## About {{metadata['repo']['name_pretty']}} + +{% if 'partials' in metadata and metadata['partials']['about'] -%} +{{ metadata['partials']['about'] }} +{% else %} +[{{ metadata['repo']['name_pretty'] }}][product-docs] {{ metadata['repo']['api_description'] }} + +See the [{{metadata['repo']['name_pretty']}} client library docs][javadocs] to learn how to +use this {{metadata['repo']['name_pretty']}} Client Library. +{% endif %} + +{% if 'partials' in metadata and metadata['partials']['custom_content'] -%} +{{ metadata['partials']['custom_content'] }} +{% endif %} + +{% if metadata['samples']|length %} +## Samples + +Samples are in the [`samples/`](https://github.com/{{ metadata['repo']['repo'] }}/tree/main/samples) directory. + +| Sample | Source Code | Try it | +| --------------------------- | --------------------------------- | ------ | +{% for sample in metadata['samples'] %}| {{ sample.title }} | [source code](https://github.com/{{ metadata['repo']['repo'] }}/blob/main/{{ sample.file }}) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/{{ metadata['repo']['repo'] }}&page=editor&open_in_editor={{ sample.file }}) | +{% endfor %} +{% endif %} + +## Troubleshooting + +To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. + +{% if metadata['repo']['transport'] -%} +## Transport + +{% if metadata['repo']['transport'] == 'grpc' -%} +{{metadata['repo']['name_pretty']}} uses gRPC for the transport layer. +{% elif metadata['repo']['transport'] == 'http' -%} +{{metadata['repo']['name_pretty']}} uses HTTP/JSON for the transport layer. +{% elif metadata['repo']['transport'] == 'both' -%} +{{metadata['repo']['name_pretty']}} uses both gRPC and HTTP/JSON for the transport layer. +{% endif %} +{% endif -%} + +## Supported Java Versions + +Java {{ metadata['min_java_version'] }} or above is required for using this client. + +Google's Java client libraries, +[Google Cloud Client Libraries][cloudlibs] +and +[Google Cloud API Libraries][apilibs], +follow the +[Oracle Java SE support roadmap][oracle] +(see the Oracle Java SE Product Releases section). + +### For new development + +In general, new feature development occurs with support for the lowest Java +LTS version covered by Oracle's Premier Support (which typically lasts 5 years +from initial General Availability). If the minimum required JVM for a given +library is changed, it is accompanied by a [semver][semver] major release. + +Java 11 and (in September 2021) Java 17 are the best choices for new +development. + +### Keeping production systems current + +Google tests its client libraries with all current LTS versions covered by +Oracle's Extended Support (which typically lasts 8 years from initial +General Availability). + +#### Legacy support + +Google's client libraries support legacy versions of Java runtimes with long +term stable libraries that don't receive feature updates on a best efforts basis +as it may not be possible to backport all patches. + +Google provides updates on a best efforts basis to apps that continue to use +Java 7, though apps might need to upgrade to current versions of the library +that supports their JVM. + +#### Where to find specific information + +The latest versions and the supported Java versions are identified on +the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` +and on [google-cloud-java][g-c-j]. + +## Versioning + +{% if 'partials' in metadata and metadata['partials']['versioning'] -%} +{{ metadata['partials']['versioning'] }} +{% else %} +This library follows [Semantic Versioning](http://semver.org/). + +{% if metadata['repo']['release_level'] in ['preview'] %} +It is currently in major version zero (``0.y.z``), which means that anything may change at any time +and the public API should not be considered stable. +{% endif %}{% endif %} + +## Contributing + +{% if 'partials' in metadata and metadata['partials']['contributing'] -%} +{{ metadata['partials']['contributing'] }} +{% else %} +Contributions to this library are always welcome and highly encouraged. + +See [CONTRIBUTING][contributing] for more information how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in +this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more +information. +{% endif %} + +## License + +Apache 2.0 - See [LICENSE][license] for more information. + +## CI Status + +Java Version | Status +------------ | ------{% if metadata['min_java_version'] <= 7 %} +Java 7 | [![Kokoro CI][kokoro-badge-image-1]][kokoro-badge-link-1]{% endif %} +Java 8 | [![Kokoro CI][kokoro-badge-image-2]][kokoro-badge-link-2] +Java 8 OSX | [![Kokoro CI][kokoro-badge-image-3]][kokoro-badge-link-3] +Java 8 Windows | [![Kokoro CI][kokoro-badge-image-4]][kokoro-badge-link-4] +Java 11 | [![Kokoro CI][kokoro-badge-image-5]][kokoro-badge-link-5] + +Java is a registered trademark of Oracle and/or its affiliates. + +[product-docs]: {{metadata['repo']['product_documentation']}} +[javadocs]: {{metadata['repo']['client_documentation']}} +[kokoro-badge-image-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java7.svg +[kokoro-badge-link-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java7.html +[kokoro-badge-image-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8.svg +[kokoro-badge-link-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8.html +[kokoro-badge-image-3]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-osx.svg +[kokoro-badge-link-3]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-osx.html +[kokoro-badge-image-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-win.svg +[kokoro-badge-link-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-win.html +[kokoro-badge-image-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.svg +[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.html +[stability-image]: https://img.shields.io/badge/stability-{% if metadata['repo']['release_level'] == 'stable' %}stable-green{% elif metadata['repo']['release_level'] == 'preview' %}preview-yellow{% else %}unknown-red{% endif %} +[maven-version-image]: https://img.shields.io/maven-central/v/{{ group_id }}/{{ artifact_id }}.svg +[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ version }} +[authentication]: https://github.com/googleapis/google-cloud-java#authentication +[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes +[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles +[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy +[developer-console]: https://console.developers.google.com/ +[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[cloud-cli]: https://cloud.google.com/cli +[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md +[contributing]: https://github.com/{{metadata['repo']['repo']}}/blob/main/CONTRIBUTING.md +[code-of-conduct]: https://github.com/{{metadata['repo']['repo']}}/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[license]: https://github.com/{{metadata['repo']['repo']}}/blob/main/LICENSE +{% if metadata['repo']['requires_billing'] %}[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing{% endif %} +{% if metadata['repo']['api_id'] %}[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid={{ metadata['repo']['api_id'] }}{% endif %} +[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png + +[semver]: https://semver.org/ +[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained +[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries +[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html +[g-c-j]: http://github.com/googleapis/google-cloud-java diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/SECURITY.md b/hermetic_build/library_generation/owlbot/templates/java_library/SECURITY.md new file mode 100644 index 0000000000..8b58ae9c01 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). + +The Google Security Team will respond within 5 working days of your report on g.co/vulnz. + +We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/java.header b/hermetic_build/library_generation/owlbot/templates/java_library/java.header new file mode 100644 index 0000000000..d0970ba7d3 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/java.header @@ -0,0 +1,15 @@ +^/\*$ +^ \* Copyright \d\d\d\d,? Google (Inc\.|LLC)$ +^ \*$ +^ \* Licensed under the Apache License, Version 2\.0 \(the "License"\);$ +^ \* you may not use this file except in compliance with the License\.$ +^ \* You may obtain a copy of the License at$ +^ \*$ +^ \*[ ]+https?://www.apache.org/licenses/LICENSE-2\.0$ +^ \*$ +^ \* Unless required by applicable law or agreed to in writing, software$ +^ \* distributed under the License is distributed on an "AS IS" BASIS,$ +^ \* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied\.$ +^ \* See the License for the specific language governing permissions and$ +^ \* limitations under the License\.$ +^ \*/$ diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/license-checks.xml b/hermetic_build/library_generation/owlbot/templates/java_library/license-checks.xml new file mode 100644 index 0000000000..6597fced80 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/license-checks.xml @@ -0,0 +1,10 @@ + + + + + + + + diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/renovate.json b/hermetic_build/library_generation/owlbot/templates/java_library/renovate.json new file mode 100644 index 0000000000..cb92a78173 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/renovate.json @@ -0,0 +1,128 @@ +{% if migrated_split_repo %} +{ + "enabled": false, +{% else %} +{ +{% endif %} + "extends": [ + ":separateMajorReleases", + ":combinePatchMinorReleases", + ":ignoreUnstable", + ":prImmediately", + ":updateNotScheduled", + ":automergeDisabled", + ":ignoreModulesAndTests", + ":maintainLockFilesDisabled", + ":autodetectPinVersions" + ], + "ignorePaths": [ + ".kokoro/requirements.txt", + ".github/workflows/approve-readme.yaml", + ".github/workflows/ci.yaml", + ".github/workflows/renovate_config_check.yaml", + ".github/workflows/samples.yaml" + ], + "customManagers": [ + { + "customType": "regex", + "fileMatch": [ + "^.kokoro/presubmit/graalvm-native.*.cfg$" + ], + "matchStrings": [ + "value: \"gcr.io/cloud-devrel-public-resources/graalvm.*:(?.*?)\"" + ], + "depNameTemplate": "com.google.cloud:sdk-platform-java-config", + "datasourceTemplate": "maven" + }, + { + "customType": "regex", + "fileMatch": [ + "^.github/workflows/unmanaged_dependency_check.yaml$" + ], + "matchStrings": [ + "uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v(?.+?)\\n" + ], + "depNameTemplate": "com.google.cloud:sdk-platform-java-config", + "datasourceTemplate": "maven" + }, + { + "fileMatch": [ + "^.github/workflows/hermetic_library_generation.yaml$" + ], + "matchStrings": [ + "uses: googleapis/sdk-platform-java/.github/scripts@v(?.+?)\\n" + ], + "depNameTemplate": "com.google.api:gapic-generator-java", + "datasourceTemplate": "maven" + } + ], + "packageRules": [ + { + "packagePatterns": [ + "^com.google.guava:" + ], + "versionScheme": "docker" + }, + { + "packagePatterns": [ + "*" + ], + "semanticCommitType": "deps", + "semanticCommitScope": null + }, + { + "packagePatterns": [ + "^org.apache.maven", + "^org.jacoco:", + "^org.codehaus.mojo:", + "^org.sonatype.plugins:", + "^com.coveo:", + "^com.google.cloud:google-cloud-shared-config" + ], + "semanticCommitType": "build", + "semanticCommitScope": "deps" + }, + { + "packagePatterns": [ + "^{{metadata['repo']['distribution_name']}}", + "^com.google.cloud:libraries-bom", + "^com.google.cloud.samples:shared-configuration" + ], + "semanticCommitType": "chore", + "semanticCommitScope": "deps" + }, + { + "packagePatterns": [ + "^junit:junit", + "^com.google.truth:truth", + "^org.mockito:mockito-core", + "^org.objenesis:objenesis", + "^com.google.cloud:google-cloud-conformance-tests", + "^org.graalvm.buildtools:junit-platform-native" + ], + "semanticCommitType": "test", + "semanticCommitScope": "deps" + }, + { + "packagePatterns": [ + "^com.google.cloud:google-cloud-" + ], + "ignoreUnstable": false + }, + { + "packagePatterns": [ + "^com.fasterxml.jackson.core" + ], + "groupName": "jackson dependencies" + }, + { + "matchPackagePatterns": [ + "^com.google.api:gapic-generator-java", + "^com.google.cloud:sdk-platform-java-config" + ], + "groupName": "SDK platform Java dependencies" + } + ], + "semanticCommits": true, + "dependencyDashboard": true +} \ No newline at end of file diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml b/hermetic_build/library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml new file mode 100644 index 0000000000..110250d003 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml @@ -0,0 +1,86 @@ +{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} +{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} + + + 4.0.0 + com.google.cloud + {{metadata['repo']['name']}}-install-without-bom + jar + Google {{metadata['repo']['name_pretty']}} Install Without Bom + https://github.com/{{metadata['repo']['repo']}} + + + + com.google.cloud.samples + shared-configuration + 1.2.0 + + + + 1.8 + 1.8 + UTF-8 + + + + + + + {{ group_id }} + {{ artifact_id }} + {{ metadata['latest_version'] }} + + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.1.3 + test + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-snippets-source + + add-source + + + + ../snippets/src/main/java + + + + + add-snippets-tests + + add-test-source + + + + ../snippets/src/test/java + + + + + + + + diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/samples/pom.xml b/hermetic_build/library_generation/owlbot/templates/java_library/samples/pom.xml new file mode 100644 index 0000000000..5fc52b1526 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/samples/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + com.google.cloud + google-cloud-{{metadata['repo']['name']}}-samples + 0.0.1-SNAPSHOT + pom + Google {{metadata['repo']['name_pretty']}} Samples Parent + https://github.com/{{metadata['repo']['repo']}} + + Java idiomatic client for Google Cloud Platform services. + + + + + com.google.cloud.samples + shared-configuration + 1.2.2 + + + + 1.8 + 1.8 + UTF-8 + + + + install-without-bom + snapshot + snippets + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.3 + + true + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.7.0 + + true + + + + + diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml b/hermetic_build/library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml new file mode 100644 index 0000000000..62a83b440e --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml @@ -0,0 +1,85 @@ +{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} +{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} + + + 4.0.0 + com.google.cloud + {{metadata['repo']['name']}}-snapshot + jar + Google {{metadata['repo']['name_pretty']}} Snapshot Samples + https://github.com/{{metadata['repo']['repo']}} + + + + com.google.cloud.samples + shared-configuration + 1.2.0 + + + + 1.8 + 1.8 + UTF-8 + + + + + + {{ group_id }} + {{ artifact_id }} + {{ metadata['latest_version'] }} + + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.1.3 + test + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-snippets-source + + add-source + + + + ../snippets/src/main/java + + + + + add-snippets-tests + + add-test-source + + + + ../snippets/src/test/java + + + + + + + + diff --git a/hermetic_build/library_generation/owlbot/templates/java_library/samples/snippets/pom.xml b/hermetic_build/library_generation/owlbot/templates/java_library/samples/snippets/pom.xml new file mode 100644 index 0000000000..c6b9981507 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/java_library/samples/snippets/pom.xml @@ -0,0 +1,49 @@ +{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} +{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} + + + 4.0.0 + com.google.cloud + {{metadata['repo']['name']}}-snippets + jar + Google {{metadata['repo']['name_pretty']}} Snippets + https://github.com/{{metadata['repo']['repo']}} + + + + com.google.cloud.samples + shared-configuration + 1.2.0 + + + + 1.8 + 1.8 + UTF-8 + + + + + + {{ group_id }} + {{ artifact_id }} + {{ metadata['latest_version'] }} + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.1.3 + test + + + diff --git a/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 new file mode 100644 index 0000000000..ddcef5226f --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/poms/bom_pom.xml.j2 @@ -0,0 +1,41 @@ + + + 4.0.0 + {{main_module.group_id}} + {{main_module.artifact_id}}-bom + {{main_module.version}} + pom + {% if monorepo -%} + + com.google.cloud + google-cloud-pom-parent + {{ monorepo_version }} + ../../google-cloud-pom-parent/pom.xml + + {%- else -%} + + com.google.cloud + google-cloud-shared-config + 1.5.3 + + {%- endif %} + + Google {{name}} BOM + + BOM for {{name}} + + + + true + + + + {% for module in modules %} + + {{module.group_id}} + {{module.artifact_id}} + {{module.version}} + {% endfor %} + + + diff --git a/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 new file mode 100644 index 0000000000..d0ae8cd7c5 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 @@ -0,0 +1,156 @@ + + + 4.0.0 + {{module.group_id}} + {{module.artifact_id}} + {{module.version}} + jar + Google {{name}} + {%- if not monorepo %} + https://github.com/{{repo}} + {%- endif %} + {{name}} {{description}} + + {{parent_module.group_id}} + {{parent_module.artifact_id}} + {{parent_module.version}} + + + {{module.artifact_id}} + + + + io.grpc + grpc-api + + + io.grpc + grpc-stub + + + io.grpc + grpc-protobuf + + + com.google.api + api-common + + + com.google.protobuf + protobuf-java + + + com.google.api.grpc + proto-google-common-protos + +{% for module in proto_modules %} + + {{module.group_id}} + {{module.artifact_id}} + {% endfor %} + + com.google.guava + guava + + + com.google.api + gax + + + com.google.api + gax-grpc + + + com.google.api + gax-httpjson + + + com.google.api.grpc + proto-google-iam-v1 + +{%- if not monorepo %} + + com.google.api.grpc + grpc-google-common-protos + + + com.google.api.grpc + grpc-google-iam-v1 + +{%- endif %} + + org.threeten + threetenbp + + + +{%- if monorepo %} + + com.google.api.grpc + grpc-google-common-protos + test + + + com.google.api.grpc + grpc-google-iam-v1 + test + +{%- endif %} + + junit + junit + test + +{% for module in grpc_modules %} + + {{module.group_id}} + {{module.artifact_id}} + test + {% endfor %} + + + com.google.api + gax + testlib + test + + + com.google.api + gax-grpc + testlib + test + + + com.google.api + gax-httpjson + testlib + test + + + + {%- if not monorepo %} + + + java9 + + [9,) + + + + javax.annotation + javax.annotation-api + + + + + + + + + org.codehaus.mojo + flatten-maven-plugin + + + + {%- endif %} + diff --git a/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 new file mode 100644 index 0000000000..514861e7a7 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 @@ -0,0 +1,71 @@ + + 4.0.0 + {{module.group_id}} + {{module.artifact_id}} + {{module.version}} + {{module.artifact_id}} + GRPC library for {{main_module.artifact_id}} + + {{parent_module.group_id}} + {{parent_module.artifact_id}} + {{parent_module.version}} + + + + io.grpc + grpc-api + + + io.grpc + grpc-stub + + + io.grpc + grpc-protobuf + + + com.google.protobuf + protobuf-java + + + com.google.api.grpc + proto-google-common-protos + + + {{proto_module.group_id}} + {{proto_module.artifact_id}} + + + com.google.guava + guava + + + + {%- if not monorepo %} + + + java9 + + [9,) + + + + javax.annotation + javax.annotation-api + + + + + + + + + org.codehaus.mojo + flatten-maven-plugin + + + + {%- endif %} + diff --git a/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 new file mode 100644 index 0000000000..dcf922340e --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/poms/parent_pom.xml.j2 @@ -0,0 +1,51 @@ + + + 4.0.0 + {{main_module.group_id}} + {{main_module.artifact_id}}-parent + pom + {{main_module.version}} + Google {{name}} Parent + + Java idiomatic client for Google Cloud Platform services. + + + {% if monorepo -%} + + com.google.cloud + google-cloud-jar-parent + {{ monorepo_version }} + ../google-cloud-jar-parent/pom.xml + + {%- else -%} + + com.google.cloud + google-cloud-shared-config + 1.5.3 + + {%- endif %} + + + UTF-8 + UTF-8 + github + {{main_module.artifact_id}}-parent + + + + +{% for module in modules %} + {{module.group_id}} + {{module.artifact_id}} + {{module.version}} + +{% endfor %} + + + + +{% for module in modules %} {{module.artifact_id}} +{% endfor %} {{main_module.artifact_id}}-bom + + + diff --git a/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 b/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 new file mode 100644 index 0000000000..886cd02663 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/poms/proto_pom.xml.j2 @@ -0,0 +1,48 @@ + + 4.0.0 + {{module.group_id}} + {{module.artifact_id}} + {{module.version}} + {{module.artifact_id}} + Proto library for {{main_module.artifact_id}} + + {{parent_module.group_id}} + {{parent_module.artifact_id}} + {{parent_module.version}} + + + + com.google.protobuf + protobuf-java + + + com.google.api.grpc + proto-google-common-protos + + + com.google.api.grpc + proto-google-iam-v1 + + + com.google.api + api-common + + + com.google.guava + guava + + + + {%- if not monorepo %} + + + + org.codehaus.mojo + flatten-maven-plugin + + + + {%- endif %} + diff --git a/hermetic_build/library_generation/owlbot/templates/poms/versions.txt.j2 b/hermetic_build/library_generation/owlbot/templates/poms/versions.txt.j2 new file mode 100644 index 0000000000..2ebaf85d34 --- /dev/null +++ b/hermetic_build/library_generation/owlbot/templates/poms/versions.txt.j2 @@ -0,0 +1,4 @@ +# Format: +# module:released-version:current-version +{% for module in modules %} +{{module.artifact_id}}:{% if module.release_version %}{{module.release_version}}{% else %}{{module.version}}{% endif %}:{{module.version}}{% endfor %} diff --git a/hermetic_build/library_generation/postprocess_library.sh b/hermetic_build/library_generation/postprocess_library.sh new file mode 100755 index 0000000000..eeec07156e --- /dev/null +++ b/hermetic_build/library_generation/postprocess_library.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# +# Main functions to interact with owlbot post-processor + +# Runs the java owlbot post-processor. The resulting post-processed +# library gets stored in the $postprocessing_target argument +# Arguments +# 1 - postprocessing_target: path where the postprocessor will run. This folder +# has the following requirements +# - a .repo-metadata.json file must be present +# - an owlbot.py file must be present +# - an .OwlBot-hermetic.yaml file must be present +# 2 - preprocessed_sources_path: used to transfer the raw grpc, proto and gapic +# libraries into the postprocessing_target via copy-code +# 3 - versions_file: path to file containing versions to be applied to the poms +# 4 - owlbot_cli_source_folder: alternative folder with a structure exactly like +# googleapis-gen. It will be used instead of preprocessed_sources_path if +# 5 - is_monorepo: whether this library is a monorepo, which implies slightly +# different logic +# 6 - libraries_bom_version: used by our implementation of owlbot to render the +# readme +# 7 - library_version: used by our implementation of owlbot to render the +# readme +set -exo pipefail +scripts_root=$(dirname "$(readlink -f "$0")") + +postprocessing_target=$1 +preprocessed_sources_path=$2 +versions_file=$3 +owlbot_cli_source_folder=$4 +is_monorepo=$5 +libraries_bom_version=$6 +library_version=$7 +owlbot_yaml_file_name=".OwlBot-hermetic.yaml" + +source "${scripts_root}"/utils/utilities.sh + +declare -a required_inputs=("postprocessing_target" "versions_file" "is_monorepo") +for required_input in "${required_inputs[@]}"; do + if [[ -z "${!required_input}" ]]; then + echo "missing required ${required_input} argument, please specify one" + exit 1 + fi +done + +for owlbot_file in ".repo-metadata.json" "owlbot.py" "${owlbot_yaml_file_name}" +do + if [[ $(find "${postprocessing_target}" -name "${owlbot_file}" | wc -l) -eq 0 ]]; then + echo "necessary file for postprocessing '${owlbot_file}' was not found in postprocessing_target" + echo "please provide a postprocessing_target folder that is compatible with the OwlBot Java postprocessor" + exit 1 + fi +done + +if [[ -z "${owlbot_cli_source_folder}" ]]; then + owlbot_cli_source_folder=$(mktemp -d) + build_owlbot_cli_source_folder "${postprocessing_target}" "${owlbot_cli_source_folder}" "${preprocessed_sources_path}" +fi + + +# we determine the location of the .OwlBot-hermetic.yaml file by checking if the target +# folder is a monorepo folder or not +if [[ "${is_monorepo}" == "true" ]]; then + # the deep-remove-regex and deep-preserve-regex of the .OwlBot-hermetic.yaml + # files in the monorepo libraries assume that `copy-code` is run + # from the root of the monorepo. However, we call `copy-code` from inside each + # library, so a path like `/java-asset/google-.*/src` will not have + # any effect. We solve this by creating a temporary owlbot yaml with + # the patched paths. + # For example, we convert + # - "/java-asset/google-.*/src" + # to + # - "/google-.*/src" + + library_name=$(basename "${postprocessing_target}") + cat "${postprocessing_target}/${owlbot_yaml_file_name}" \ + | sed "s/- \"\/${library_name}/ - \"/" \ + > "${postprocessing_target}/.OwlBot.hermetic.yaml" + owlbot_yaml_relative_path=".OwlBot.hermetic.yaml" +else + owlbot_yaml_relative_path=".github/${owlbot_yaml_file_name}" +fi + +# Default values for running copy-code directly from host +repo_workspace="/workspace" +preprocessed_libraries_binding="${owlbot_cli_source_folder}" + +pushd "${postprocessing_target}" + +owl-bot copy-code \ + --source-repo-commit-hash=none \ + --source-repo="${owlbot_cli_source_folder}" \ + --config-file="${owlbot_yaml_relative_path}" + + +# clean the custom owlbot yaml +if [[ "${is_monorepo}" == "true" ]]; then + rm "${postprocessing_target}/.OwlBot.hermetic.yaml" +fi + +# run the postprocessor +echo 'running owl-bot post-processor' +pushd "${postprocessing_target}" +bash "${scripts_root}/owlbot/bin/entrypoint.sh" \ + "${scripts_root}" \ + "${versions_file}" \ + "${is_monorepo}" \ + "${libraries_bom_version}" \ + "${library_version}" + +popd # postprocessing_target diff --git a/hermetic_build/library_generation/setup.py b/hermetic_build/library_generation/setup.py new file mode 100755 index 0000000000..19737b9890 --- /dev/null +++ b/hermetic_build/library_generation/setup.py @@ -0,0 +1,42 @@ +""" +Package information of library_generation python scripts +""" + +from setuptools import setup + +setup( + name="library_generation", + version="0.1", + python_requires=">=3.12", + package_dir={ + "library_generation": ".", + "synthtool": "owlbot/synthtool", + }, + install_requires=[ + "attrs==24.2.0", + "click==8.1.7", + "common==0.1", # local package + "GitPython==3.1.43", + "jinja2==3.1.4", + "lxml==5.3.0", + "PyYAML==6.0.2", + "requests==2.32.3", + "requests-mock==1.12.1", + ], + package_data={ + "library_generation": [ + "generate_library.sh", + "postprocess_library.sh", + "utils/utilities.sh", + "templates/*.j2", + "gapic-generator-java-wrapper", + "owlbot/bin/*.sh", + "owlbot/src/*.py", + "owlbot/src/poms/*.py", + "owlbot/templates/clirr/*.j2", + "owlbot/templates/poms/*.j2", + "owlbot/templates/java_library/**/*", + ], + "synthtool": ["owlbot/synthtool/**/*"], + }, +) diff --git a/hermetic_build/library_generation/templates/gapic-libraries-bom.xml.j2 b/hermetic_build/library_generation/templates/gapic-libraries-bom.xml.j2 new file mode 100644 index 0000000000..45dbdf42ce --- /dev/null +++ b/hermetic_build/library_generation/templates/gapic-libraries-bom.xml.j2 @@ -0,0 +1,37 @@ + + + 4.0.0 + com.google.cloud + gapic-libraries-bom + pom + {{ monorepo_version }} + Google Cloud Java BOM + + BOM for the libraries in google-cloud-java repository. Users should not + depend on this artifact explicitly because this BOM is an implementation + detail of the Libraries BOM. + + + + google-cloud-pom-parent + com.google.cloud + {{ monorepo_version }} + ../google-cloud-pom-parent/pom.xml + + + + + {%- for bom_config in bom_configs %} + + {{ bom_config.group_id }} + {{ bom_config.artifact_id }} + {{ bom_config.version }} + {%- if bom_config.is_import %} + pom + import + {%- endif %} + + {%- endfor %} + + + diff --git a/hermetic_build/library_generation/templates/owlbot.py.j2 b/hermetic_build/library_generation/templates/owlbot.py.j2 new file mode 100644 index 0000000000..b3a7dee5a4 --- /dev/null +++ b/hermetic_build/library_generation/templates/owlbot.py.j2 @@ -0,0 +1,28 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import synthtool as s +{% if should_include_templates %}from synthtool.languages import java{% endif %} + + +for library in s.get_staging_dirs(): + # put any special-case replacements here + s.move(library) + +s.remove_staging_dirs() +{% if should_include_templates %}java.common_templates(monorepo=True, {% if template_excludes %}excludes=[ +{%- for exclude in template_excludes %} + "{{ exclude }}"{% if not loop.last %},{% endif %} +{%- endfor %} +]{% endif %}){% endif %} diff --git a/hermetic_build/library_generation/templates/owlbot.yaml.monorepo.j2 b/hermetic_build/library_generation/templates/owlbot.yaml.monorepo.j2 new file mode 100644 index 0000000000..9ed63c4260 --- /dev/null +++ b/hermetic_build/library_generation/templates/owlbot.yaml.monorepo.j2 @@ -0,0 +1,36 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{% if artifact_id %} +deep-remove-regex: +- "/{{ module_name }}/grpc-google-.*/src" +- "/{{ module_name }}/proto-google-.*/src" +- "/{{ module_name }}/google-.*/src" +- "/{{ module_name }}/samples/snippets/generated" + +deep-preserve-regex: +- "/{{ module_name }}/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" + +deep-copy-regex: +- source: "/{{ proto_path }}/(v.*)/.*-java/proto-google-.*/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_id }}-$1/src" +- source: "/{{ proto_path }}/(v.*)/.*-java/grpc-google-.*/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_id }}-$1/src" +- source: "/{{ proto_path }}/(v.*)/.*-java/gapic-google-.*/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_id }}/src" +- source: "/{{ proto_path }}/(v.*)/.*-java/samples/snippets/generated" + dest: "/owl-bot-staging/{{ module_name }}/$1/samples/snippets/generated" +{%- endif %} + +api-name: {{ api_shortname }} diff --git a/hermetic_build/library_generation/templates/root-pom.xml.j2 b/hermetic_build/library_generation/templates/root-pom.xml.j2 new file mode 100644 index 0000000000..56704aa4e4 --- /dev/null +++ b/hermetic_build/library_generation/templates/root-pom.xml.j2 @@ -0,0 +1,88 @@ + + + 4.0.0 + google-cloud-java + com.google.cloud + 0.201.0 + pom + + + true + + + + gapic-libraries-bom + google-cloud-jar-parent + google-cloud-pom-parent + {%- for module in modules %} + {{ module }} + {%- endfor %} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.3 + + true + + + + + + + + release-staging-repository + + + + !gpg.executable + + + + + sonatype-nexus-snapshots + https://google.oss.sonatype.org/content/repositories/snapshots + + + sonatype-nexus-staging + https://google.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.7.0 + true + + sonatype-nexus-staging + https://google.oss.sonatype.org/ + false + + + + + + + release-non-google-oss-sonatype + + + + org.sonatype.plugins + nexus-staging-maven-plugin + + ossrh + https://oss.sonatype.org/ + + + + + + + diff --git a/hermetic_build/library_generation/tests/__init__.py b/hermetic_build/library_generation/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/cli/__init__.py b/hermetic_build/library_generation/tests/cli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py b/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py new file mode 100644 index 0000000000..b7e7088cf9 --- /dev/null +++ b/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py @@ -0,0 +1,416 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +from unittest.mock import patch, ANY +from click.testing import CliRunner +from library_generation.cli.entry_point import ( + generate, + validate_generation_config, + __generate_repo_and_pr_description_impl as generate_impl, +) +from common.model.generation_config import from_yaml + +script_dir = os.path.dirname(os.path.realpath(__file__)) +test_resource_dir = os.path.join(script_dir, "..", "resources", "test-config") + + +class EntryPointTest(unittest.TestCase): + def test_entry_point_without_config_raise_file_exception(self): + os.chdir(script_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke(generate, ["--repository-path=."]) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex( + result.exception.args[0], "generation_config.yaml does not exist." + ) + + def test_entry_point_with_baseline_without_current_raise_file_exception(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + generate, + [ + "--baseline-generation-config-path=path/to/config.yaml", + "--repository-path=.", + ], + ) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex( + result.exception.args[0], + "current_generation_config is not specified when " + "baseline_generation_config is specified.", + ) + + def test_validate_generation_config_succeeds( + self, + ): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + validate_generation_config, + [f"--generation-config-path={test_resource_dir}/generation_config.yaml"], + ) + self.assertEqual(0, result.exit_code) + + def test_validate_generation_config_with_duplicate_library_name_raise_file_exception( + self, + ): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + validate_generation_config, + [ + f"--generation-config-path={test_resource_dir}/generation_config_with_duplicate_library_name.yaml" + ], + ) + self.assertEqual(1, result.exit_code) + self.assertEqual(SystemExit, result.exc_info[0]) + self.assertRegex( + result.output, + "have the same library name", + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_non_monorepo_without_changes_triggers_full_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of non monorepos + (HW libraries). generate() should call generate_from_yaml() + with target_library_names=None in order to trigger the full generation + """ + config_path = f"{test_resource_dir}/generation_config.yaml" + self.assertFalse(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names=None, + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=None, + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_non_monorepo_without_changes_with_includes_triggers_selective_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of non monorepos + (HW libraries). + generate() should call generate_from_yaml() with + target_library_names equals includes. + """ + config_path = f"{test_resource_dir}/generation_config.yaml" + self.assertFalse(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names="cloudasset,non-existent-library", + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["cloudasset", "non-existent-library"], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_non_monorepo_with_changes_triggers_full_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of non monorepos + (HW libraries). generate() should call generate_from_yaml() + with target_library_names=None in order to trigger the full generation + """ + baseline_config_path = f"{test_resource_dir}/generation_config.yaml" + current_config_path = ( + f"{test_resource_dir}/generation_config_library_modified.yaml" + ) + self.assertFalse(from_yaml(current_config_path).is_monorepo()) + self.assertFalse(from_yaml(baseline_config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=baseline_config_path, + current_generation_config_path=current_config_path, + library_names=None, + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=None, + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_non_monorepo_with_changes_with_includes_triggers_selective_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of non monorepos + (HW libraries). + generate() should call generate_from_yaml() with + target_library_names equals includes + """ + baseline_config_path = f"{test_resource_dir}/generation_config.yaml" + current_config_path = ( + f"{test_resource_dir}/generation_config_library_modified.yaml" + ) + self.assertFalse(from_yaml(current_config_path).is_monorepo()) + self.assertFalse(from_yaml(baseline_config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=baseline_config_path, + current_generation_config_path=current_config_path, + library_names="cloudasset,non-existent-library", + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["cloudasset", "non-existent-library"], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_monorepo_with_common_protos_triggers_full_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo with + common protos. + generate() should call generate_from_yaml() with + target_library_names=None in order to trigger the full generation + """ + config_path = f"{test_resource_dir}/monorepo_with_common_protos.yaml" + self.assertTrue(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names=None, + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=None, + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo with + common protos. + target_library_names is the same as includes. + """ + config_path = f"{test_resource_dir}/monorepo_with_common_protos.yaml" + self.assertTrue(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names="iam,non-existent-library", + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["iam", "non-existent-library"], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_monorepo_without_change_does_not_trigger_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo without + common protos. + generate() should call generate_from_yaml() with + target_library_names=changed libraries which does not trigger the full + generation. + """ + config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" + self.assertTrue(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names=None, + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=[], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_monorepo_without_change_with_includes_trigger_selective_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo without + common protos. + generate() should call generate_from_yaml() with + target_library_names=changed libraries which does not trigger the full + generation. + """ + config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" + self.assertTrue(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names="asset", + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["asset"], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_monorepo_with_changed_config_without_includes_trigger_changed_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo without + common protos. + target_library_names should be the changed libraries if includes + is not specified. + """ + current_config_path = f"{test_resource_dir}/monorepo_current.yaml" + baseline_config_path = f"{test_resource_dir}/monorepo_baseline.yaml" + self.assertTrue(from_yaml(current_config_path).is_monorepo()) + self.assertTrue(from_yaml(baseline_config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=baseline_config_path, + current_generation_config_path=current_config_path, + library_names=None, + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["asset"], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_monorepo_with_changed_config_and_includes_trigger_selective_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo without + common protos. + target_library_names should be the same as include libraries, regardless + the library exists or not. + """ + current_config_path = f"{test_resource_dir}/monorepo_current.yaml" + baseline_config_path = f"{test_resource_dir}/monorepo_baseline.yaml" + self.assertTrue(from_yaml(current_config_path).is_monorepo()) + self.assertTrue(from_yaml(baseline_config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=baseline_config_path, + current_generation_config_path=current_config_path, + library_names="cloudbuild,non-existent-library", + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["cloudbuild", "non-existent-library"], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + def test_generate_monorepo_without_changed_config_without_includes_does_not_trigger_generation( + self, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo without + common protos. + target_library_names should be the changed libraries if includes + is not specified. + """ + config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" + self.assertTrue(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names=None, + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=[], + ) diff --git a/hermetic_build/library_generation/tests/compare_poms.py b/hermetic_build/library_generation/tests/compare_poms.py new file mode 100644 index 0000000000..95041651fb --- /dev/null +++ b/hermetic_build/library_generation/tests/compare_poms.py @@ -0,0 +1,131 @@ +""" +Utility to compare the contents of two XML files. +This focuses on the tree structure of both XML files, meaning that element order and whitespace will be disregarded. +The only comparison points are: element path (e.g. project/dependencies) and element text +There is a special case for `dependency`, where the maven coordinates are prepared as well +""" + +from library_generation.utils.utilities import eprint +import xml.etree.ElementTree as et +from collections import Counter +import sys +import os + +current = os.path.dirname(os.path.realpath(__file__)) +parent = os.path.dirname(current) +sys.path.append(parent) + + +def get_text_from_element(node, element_name, namespace): + """ + Convenience method to access a node's child elements via path and get + its text. + """ + child = node.find(namespace + element_name) + return child.text if child is not None else "" + + +def print_counter(counter): + """ + Convenience method to pretty print the contents of a Counter (or dict) + """ + for key, value in counter.items(): + eprint(f"{key}: {value}") + + +def append_to_element_list(node, path, elements): + """ + Recursively traverses a node tree and appends element text to a given + `elements` array. If the element tag is `dependency` + then the maven coordinates for its children will be computed as well + """ + namespace_start, namespace_end, tag_name = node.tag.rpartition("}") + namespace = namespace_start + namespace_end + if tag_name == "dependency": + group_id = get_text_from_element(node, "groupId", namespace) + artifact_id = get_text_from_element(node, "artifactId", namespace) + artifact_str = "" + artifact_str += group_id + artifact_str += ":" + artifact_id + elements.append(path + "/" + tag_name + "=" + artifact_str) + if node.text and len(node.text.strip()) > 0: + elements.append(path + "/" + tag_name + "=" + node.text) + + if tag_name == "version": + # versions may be yet to be processed, we disregard them + return elements + + for child in node: + child_path = path + "/" + tag_name + append_to_element_list(child, child_path, elements) + + return elements + + +def compare_xml(expected, actual, print_trees): + """ + compares two XMLs for content differences + the argument print_whole_trees determines if both trees should be printed + """ + try: + expected_tree = et.parse(expected) + actual_tree = et.parse(actual) + except et.ParseError as e: + eprint(f"Error parsing XML") + raise e + except FileNotFoundError as e: + eprint(f"Error reading file") + raise e + + expected_elements = [] + actual_elements = [] + + append_to_element_list(expected_tree.getroot(), "/", expected_elements) + append_to_element_list(actual_tree.getroot(), "/", actual_elements) + + expected_counter = Counter(expected_elements) + actual_counter = Counter(actual_elements) + intersection = expected_counter & actual_counter + only_in_expected = expected_counter - intersection + only_in_actual = actual_counter - intersection + if print_trees: + eprint("expected") + print_counter(actual_counter) + eprint("actual") + print_counter(expected_counter) + if len(only_in_expected) > 0 or len(only_in_actual) > 0: + eprint("only in " + expected) + print_counter(only_in_expected) + eprint("only in " + actual) + print_counter(only_in_actual) + return True + return False + + +def compare_pom_in_subdir(base_dir: str, subdir: str) -> bool: + golden = os.path.join(base_dir, subdir, "pom-golden.xml") + pom = os.path.join(base_dir, subdir, "pom.xml") + return compare_xml( + golden, + pom, + False, + ) + + +if __name__ == "__main__": + if len(sys.argv) != 4: + eprint( + "Usage: python compare_xml.py " + ) + sys.exit(1) + + file1 = sys.argv[1] + file2 = sys.argv[2] + print_whole_trees = sys.argv[3] + has_diff = compare_xml(file1, file2, print_whole_trees) + + if has_diff: + eprint(f"The poms are different") + sys.exit(1) + eprint("The XML files are the same.") + sys.exit(0) diff --git a/hermetic_build/library_generation/tests/generate_library_unit_tests.py b/hermetic_build/library_generation/tests/generate_library_unit_tests.py new file mode 100644 index 0000000000..7bc14e3e20 --- /dev/null +++ b/hermetic_build/library_generation/tests/generate_library_unit_tests.py @@ -0,0 +1,109 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import subprocess +import unittest +import os +from library_generation.utils.utilities import ( + run_process_and_print_output as bash_call, + run_process_and_get_output_string as get_bash_call_output, +) + +script_dir = os.path.dirname(os.path.realpath(__file__)) + + +class GenerateLibraryUnitTests(unittest.TestCase): + """ + Confirms the correct behavior of `library_generation/utils/utilities.sh`. + + Note that there is an already existing, shell-based, test suite for + generate_library.sh, but these tests will soon be transferred to this one as + an effort to unify the implementation of the Hermetic Build scripts as + python-only. New tests for `utilities.sh` should be added in this file. + """ + + TEST_ARCHITECTURE = "linux-x86_64" + + def setUp(self): + # we create a simulated home folder that has a fake generator jar + # in its well-known location + self.simulated_home = get_bash_call_output("mktemp -d") + bash_call(f"mkdir {self.simulated_home}/.library_generation") + bash_call( + f"touch {self.simulated_home}/.library_generation/gapic-generator-java.jar" + ) + + # We create a per-test directory where all output files will be created into. + # Each folder will be deleted after its corresponding test finishes. + test_dir = get_bash_call_output("mktemp -d") + self.output_folder = self._run_command_and_get_sdout( + "get_output_folder", + cwd=test_dir, + ) + bash_call(f"mkdir {self.output_folder}") + + def tearDown(self): + bash_call(f"rm -rdf {self.simulated_home}") + + def _run_command(self, command, **kwargs): + env = os.environ.copy() + env["HOME"] = self.simulated_home + if "cwd" not in kwargs: + kwargs["cwd"] = self.output_folder + return bash_call( + [ + "bash", + "-exc", + f"source {script_dir}/../utils/utilities.sh " + f"&& {command}", + ], + exit_on_fail=False, + env=env, + **kwargs, + ) + + def _run_command_and_get_sdout(self, command, **kwargs): + return self._run_command( + command, stderr=subprocess.PIPE, **kwargs + ).stdout.decode()[:-1] + + def test_get_grpc_version_with_no_env_var_fails(self): + # the absence of DOCKER_GRPC_VERSION will make this function to fail + result = self._run_command("get_grpc_version") + self.assertEqual(1, result.returncode) + self.assertRegex(result.stdout.decode(), "DOCKER_GRPC_VERSION is not set") + + def test_get_protoc_version_with_no_env_var_fails(self): + # the absence of DOCKER_PROTOC_VERSION will make this function to fail + result = self._run_command("get_protoc_version") + self.assertEqual(1, result.returncode) + self.assertRegex(result.stdout.decode(), "DOCKER_PROTOC_VERSION is not set") + + def test_download_tools_without_baked_generator_fails(self): + # This test has the same structure as + # download_tools_succeed_with_baked_protoc, but meant for + # gapic-generator-java. + + test_protoc_version = "1.64.0" + test_grpc_version = "1.64.0" + jar_location = ( + f"{self.simulated_home}/.library_generation/gapic-generator-java.jar" + ) + # we expect the function to fail because the generator jar is not found in + # its well-known location. To achieve this, we temporarily remove the fake + # generator jar + bash_call(f"rm {jar_location}") + result = self._run_command( + f"download_tools {test_protoc_version} {test_grpc_version} {self.TEST_ARCHITECTURE}" + ) + self.assertEqual(1, result.returncode) + self.assertRegex(result.stdout.decode(), "Please configure your environment") diff --git a/hermetic_build/library_generation/tests/generate_library_unit_tests.sh b/hermetic_build/library_generation/tests/generate_library_unit_tests.sh new file mode 100755 index 0000000000..639abd8677 --- /dev/null +++ b/hermetic_build/library_generation/tests/generate_library_unit_tests.sh @@ -0,0 +1,314 @@ +#!/usr/bin/env bash + +set -xeo pipefail + +# Unit tests against ../utilities.sh +script_dir=$(dirname "$(readlink -f "$0")") +source "${script_dir}"/test_utilities.sh + +# we simulate a properly prepared environment (i.e. generator jar in its +# well-known location). Tests confirming the opposite case will make sure this +# environment is restored +readonly SIMULATED_HOME=$(mktemp -d) +mkdir "${SIMULATED_HOME}/.library_generation" +touch "${SIMULATED_HOME}/.library_generation/gapic-generator-java.jar" +touch "${SIMULATED_HOME}/.library_generation/google-java-format.jar" +HOME="${SIMULATED_HOME}" source "${script_dir}"/../utils/utilities.sh + +# Unit tests +extract_folder_name_test() { + local path="google/cloud/aiplatform/v1/google-cloud-aiplatform-v1-java" + local folder_name + folder_name=$(extract_folder_name "${path}") + assertEquals "google-cloud-aiplatform-v1-java" "${folder_name}" +} + +get_grpc_version_succeed_docker_env_var_test() { + local version_with_docker + local version_without_docker + export DOCKER_GRPC_VERSION="9.9.9" + # get_grpc_version should prioritize DOCKER_GRPC_VERSION + version_with_docker=$(get_grpc_version) + assertEquals "${DOCKER_GRPC_VERSION}" "${version_with_docker}" + unset DOCKER_GRPC_VERSION +} + +get_protoc_version_succeed_docker_env_var_test() { + local version_with_docker + local version_without_docker + export DOCKER_PROTOC_VERSION="9.9.9" + # get_protoc_version should prioritize DOCKER_PROTOC_VERSION + version_with_docker=$(get_protoc_version) + assertEquals "${DOCKER_PROTOC_VERSION}" "${version_with_docker}" + unset DOCKER_PROTOC_VERSION +} + +get_gapic_opts_with_rest_test() { + local proto_path="${script_dir}/resources/gapic_options" + local transport="grpc" + local rest_numeric_enums="true" + local gapic_opts + gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "")" + assertEquals \ + "transport=grpc,rest-numeric-enums,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \ + "${gapic_opts}" +} + +get_gapic_opts_without_rest_test() { + local proto_path="${script_dir}/resources/gapic_options" + local transport="grpc" + local rest_numeric_enums="false" + local gapic_opts + gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "")" + assertEquals \ + "transport=grpc,,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \ + "$gapic_opts" +} + +get_gapic_opts_with_non_default_test() { + local proto_path="${script_dir}/resources/gapic_options" + local transport="grpc" + local rest_numeric_enums="false" + local gapic_opts + gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "${proto_path}/example_gapic.yaml" "${proto_path}/example_grpc_service_config.json" "${proto_path}/example.yaml")" + assertEquals \ + "transport=grpc,,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \ + "$gapic_opts" +} + +remove_grpc_version_test() { + local destination_path="${script_dir}/resources/gapic_options" + cp "${destination_path}/QueryServiceGrpc_copy.java" "${destination_path}/QueryServiceGrpc.java" + remove_grpc_version "${destination_path}" + local res=0 + if ! grep -q 'value = "by gRPC proto compiler",' "${destination_path}/QueryServiceGrpc.java"; then + echo "Error: grpc version is not removed." + res=1 + fi + + assertEquals 0 $((res)) + rm "${destination_path}/QueryServiceGrpc.java" +} + +download_protoc_succeed_with_valid_version_linux_test() { + download_protoc "23.2" "linux-x86_64" + assertFileOrDirectoryExists "protoc-23.2" + rm -rf "protoc-23.2" +} + +download_protoc_succeed_with_valid_version_macos_test() { + download_protoc "23.2" "osx-x86_64" + assertFileOrDirectoryExists "protoc-23.2" + rm -rf "protoc-23.2" "google" +} + +download_protoc_failed_with_invalid_version_linux_test() { + local res=0 + $(download_protoc "22.99" "linux-x86_64") || res=$? + assertEquals 1 $((res)) +} + +download_protoc_failed_with_invalid_arch_test() { + local res=0 + $(download_protoc "23.2" "customized-x86_64") || res=$? + assertEquals 1 $((res)) +} + +download_tools_succeed_with_baked_protoc() { + # This mimics a docker container scenario. + # This test consists of creating an empty /tmp/.../protoc-99.99/bin folder and map + # it to the DOCKER_PROTOC_LOCATION env var (which is treated specially in the + # `download_tools` function). If `DOCKER_PROTOC_VERSION` matches exactly as + # the version passed to `download_protoc`, then we will not download protoc + # but simply have the variable `protoc_path` pointing to DOCKER_PROTOC_LOCATION + # (which we manually created in this test) + export DOCKER_PROTOC_LOCATION=$(mktemp -d) + export DOCKER_PROTOC_VERSION="99.99" + export output_folder=$(get_output_folder) + mkdir "${output_folder}" + local protoc_bin_folder="${DOCKER_PROTOC_LOCATION}/protoc-99.99/bin" + mkdir -p "${protoc_bin_folder}" + + local test_grpc_version="1.64.0" + # we expect download_tools to decide to use DOCKER_PROTOC_LOCATION because + # the protoc version we want to download is the same as DOCKER_PROTOC_VERSION. + # Note that `protoc_bin_folder` is just the expected formatted value that + # download_tools will format using DOCKER_PROTOC_VERSION (via + # download_protoc). + download_tools "99.99" "${test_grpc_version}" "linux-x86_64" + assertEquals "${protoc_bin_folder}" "${protoc_path}" + + rm -rdf "${output_folder}" + unset DOCKER_PROTOC_LOCATION + unset DOCKER_PROTOC_VERSION + unset output_folder + unset protoc_path +} + +download_tools_succeed_with_baked_grpc() { + # This test has the same structure as + # download_tools_succeed_with_baked_protoc, but meant for the grpc plugin. + export DOCKER_GRPC_LOCATION=$(mktemp -d) + export DOCKER_GRPC_VERSION="99.99" + export output_folder=$(get_output_folder) + mkdir "${output_folder}" + + local test_protoc_version="1.64.0" + # we expect download_tools to decide to use DOCKER_GRPC_LOCATION because + # the protoc version we want to download is the same as DOCKER_GRPC_VERSION + download_tools "${test_protoc_version}" "99.99" "linux-x86_64" + assertEquals "${DOCKER_GRPC_LOCATION}" "${grpc_path}" + + rm -rdf "${output_folder}" + unset DOCKER_GRPC_LOCATION + unset DOCKER_GRPC_VERSION + unset output_folder + unset grpc_path +} + +download_grpc_plugin_succeed_with_valid_version_linux_test() { + download_grpc_plugin "1.55.1" "linux-x86_64" + assertFileOrDirectoryExists "protoc-gen-grpc-java-1.55.1-linux-x86_64.exe" + rm "protoc-gen-grpc-java-1.55.1-linux-x86_64.exe" +} + +download_grpc_plugin_succeed_with_valid_version_macos_test() { + download_grpc_plugin "1.55.1" "osx-x86_64" + assertFileOrDirectoryExists "protoc-gen-grpc-java-1.55.1-osx-x86_64.exe" + rm "protoc-gen-grpc-java-1.55.1-osx-x86_64.exe" +} + +download_grpc_plugin_failed_with_invalid_version_linux_test() { + local res=0 + $(download_grpc_plugin "0.99.0" "linux-x86_64") || res=$? + assertEquals 1 $((res)) +} + +download_grpc_plugin_failed_with_invalid_arch_test() { + local res=0 + $(download_grpc_plugin "1.55.1" "customized-x86_64") || res=$? + assertEquals 1 $((res)) +} + +generate_library_failed_with_invalid_generator_version() { + local destination="google-cloud-alloydb-v1-java" + local res=0 + cd "${script_dir}/resources" + bash "${script_dir}"/../generate_library.sh \ + -p google/cloud/alloydb/v1 \ + -d ../"${destination}" \ + --protoc_version 23.2 \ + --grpc_version 1.55.1 \ + --transport grpc+rest \ + --rest_numeric_enums true || res=$? + assertEquals 1 $((res)) + # still need to clean up potential downloaded tooling. + cleanup "${destination}" +} + +generate_library_failed_with_invalid_protoc_version() { + local destination="google-cloud-alloydb-v1-java" + local res=0 + cd "${script_dir}/resources" + bash "${script_dir}"/../generate_library.sh \ + -p google/cloud/alloydb/v1 \ + -d ../"${destination}" \ + --protoc_version 22.99 \ + --grpc_version 1.55.1 \ + --transport grpc+rest \ + --rest_numeric_enums true || res=$? + assertEquals 1 $((res)) + # still need to clean up potential downloaded tooling. + cleanup "${destination}" +} + +generate_library_failed_with_invalid_grpc_version() { + local destination="google-cloud-alloydb-v1-java" + local res=0 + cd "${script_dir}/resources" + bash "${script_dir}"/../generate_library.sh \ + -p google/cloud/alloydb/v1 \ + -d ../output/"${destination}" \ + --grpc_version 0.99.0 \ + --transport grpc+rest \ + --rest_numeric_enums true || res=$? + assertEquals 1 $((res)) + # still need to clean up potential downloaded tooling. + cleanup "${destination}" +} + +copy_directory_if_exists_valid_folder_succeeds() { + local source_folder="${script_dir}/resources" + local destination="${script_dir}/test_destination_folder" + mkdir -p "${destination}" + copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" + n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) + rm -rdf "${destination}" + assertEquals 1 ${n_matching_folders} +} + +copy_directory_if_exists_invalid_folder_does_not_copy() { + local source_folder="${script_dir}/non-existent" + local destination="${script_dir}/test_destination_folder" + mkdir -p "${destination}" + copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" + n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) || res=$? + rm -rdf "${destination}" + assertEquals 0 ${n_matching_folders} +} + +get_proto_path_from_preprocessed_sources_valid_library_succeeds() { + local sources="${script_dir}/resources/proto_path_library" + local proto_path=$(get_proto_path_from_preprocessed_sources "${sources}") + assertEquals "google/cloud/test/v1" ${proto_path} +} + +get_proto_path_from_preprocessed_sources_empty_library_fails() { + local sources=$(mktemp -d) + ( + get_proto_path_from_preprocessed_sources "${sources}" + ) || res=$? + assertEquals 1 ${res} +} + +get_proto_path_from_preprocessed_sources_multiple_proto_dirs_fails() { + local sources="${script_dir}/resources/proto_path_library_multiple_protos" + ( + get_proto_path_from_preprocessed_sources "${sources}" + ) || res=$? + assertEquals 1 ${res} +} + +# Execute tests. +# One line per test. +test_list=( + extract_folder_name_test + get_grpc_version_succeed_docker_env_var_test + get_protoc_version_succeed_docker_env_var_test + get_gapic_opts_with_rest_test + get_gapic_opts_without_rest_test + get_gapic_opts_with_non_default_test + remove_grpc_version_test + download_protoc_succeed_with_valid_version_linux_test + download_protoc_succeed_with_valid_version_macos_test + download_protoc_failed_with_invalid_version_linux_test + download_protoc_failed_with_invalid_arch_test + download_tools_succeed_with_baked_protoc + download_tools_succeed_with_baked_grpc + download_grpc_plugin_succeed_with_valid_version_linux_test + download_grpc_plugin_succeed_with_valid_version_macos_test + download_grpc_plugin_failed_with_invalid_version_linux_test + download_grpc_plugin_failed_with_invalid_arch_test + generate_library_failed_with_invalid_generator_version + generate_library_failed_with_invalid_protoc_version + generate_library_failed_with_invalid_grpc_version + copy_directory_if_exists_valid_folder_succeeds + copy_directory_if_exists_invalid_folder_does_not_copy + get_proto_path_from_preprocessed_sources_valid_library_succeeds + get_proto_path_from_preprocessed_sources_empty_library_fails + get_proto_path_from_preprocessed_sources_multiple_proto_dirs_fails +) + +pushd "${script_dir}" +execute_tests "${test_list[@]}" +popd diff --git a/hermetic_build/library_generation/tests/generate_repo_unit_tests.py b/hermetic_build/library_generation/tests/generate_repo_unit_tests.py new file mode 100644 index 0000000000..3d81e2a235 --- /dev/null +++ b/hermetic_build/library_generation/tests/generate_repo_unit_tests.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.generate_repo import get_target_libraries +from common.model.generation_config import GenerationConfig +from common.model.library_config import LibraryConfig + + +class GenerateRepoTest(unittest.TestCase): + def test_get_target_library_returns_selected_libraries(self): + one_library = GenerateRepoTest.__get_an_empty_library_config() + one_library.api_shortname = "one_library" + another_library = GenerateRepoTest.__get_an_empty_library_config() + another_library.api_shortname = "another_library" + config = GenerateRepoTest.__get_an_empty_generation_config() + config.libraries.extend([one_library, another_library]) + target_libraries = get_target_libraries(config, ["another_library"]) + self.assertEqual([another_library], target_libraries) + + def test_get_target_library_given_null_returns_all_libraries(self): + one_library = GenerateRepoTest.__get_an_empty_library_config() + one_library.api_shortname = "one_library" + another_library = GenerateRepoTest.__get_an_empty_library_config() + another_library.api_shortname = "another_library" + config = GenerateRepoTest.__get_an_empty_generation_config() + config.libraries.extend([one_library, another_library]) + target_libraries = get_target_libraries(config) + self.assertEqual([one_library, another_library], target_libraries) + + def test_get_target_library_given_an_non_existent_library_returns_only_existing_libraries( + self, + ): + one_library = GenerateRepoTest.__get_an_empty_library_config() + one_library.api_shortname = "one_library" + another_library = GenerateRepoTest.__get_an_empty_library_config() + another_library.api_shortname = "another_library" + config = GenerateRepoTest.__get_an_empty_generation_config() + config.libraries.extend([one_library, another_library]) + target_libraries = get_target_libraries( + config, ["one_library", "another_library", "non_existent_library"] + ) + self.assertEqual([one_library, another_library], target_libraries) + + @staticmethod + def __get_an_empty_generation_config() -> GenerationConfig: + return GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + libraries=[], + ) + + @staticmethod + def __get_an_empty_library_config() -> LibraryConfig: + return LibraryConfig( + api_shortname="", + name_pretty="", + api_description="", + product_documentation="", + gapic_configs=[], + ) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py new file mode 100644 index 0000000000..b189c9481b --- /dev/null +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -0,0 +1,394 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from click.testing import CliRunner +import difflib +import json +import tempfile +from filecmp import cmp +from filecmp import dircmp +from git import Repo +import os +import shutil +import subprocess +import unittest +from distutils.dir_util import copy_tree +from distutils.file_util import copy_file +from pathlib import Path +from library_generation.cli.generate_release_note import ( + generate as generate_pr_description, +) +from common.model.generation_config import GenerationConfig +from common.model.generation_config import from_yaml +from library_generation.tests.compare_poms import compare_xml +from library_generation.utils.utilities import sh_util as shell_call + + +script_dir = os.path.dirname(os.path.realpath(__file__)) +config_dir = os.path.join(script_dir, "resources", "integration") +golden_dir = os.path.join(config_dir, "golden") +generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") +repo_root_dir = os.path.join(script_dir, "..", "..") +build_file = os.path.join( + repo_root_dir, ".cloudbuild", "library_generation", "library_generation.Dockerfile" +) +image_tag = "test-image:latest" +repo_prefix = "https://github.com/googleapis" +output_dir = shell_call("get_output_folder") +# this map tells which branch of each repo should we use for our diff tests +commitish_map = { + "google-cloud-java": "chore/test-hermetic-build", + "java-bigtable": "chore/test-hermetic-build", +} +baseline_config_name = "baseline_generation_config.yaml" +current_config_name = "current_generation_config.yaml" +googleapis_commitish = "113a378d5aad5018876ec0a8cbfd4d6a4f746809" +# This variable is used to override the jar created by building the image +# with our own downloaded jar in order to lock the integration test to use +# a constant version specified in +# library_generation/test/resources/integration/test_generator_coordinates +# This allows us to decouple the generation workflow testing with what the +# generator jar will actually generate. +# See library_generation/DEVELOPMENT.md ("The Hermetic Build's +# well-known folder"). +WELL_KNOWN_GENERATOR_JAR_FILENAME = "gapic-generator-java.jar" + + +class IntegrationTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.__download_generator_jar(coordinates_file=generator_jar_coordinates_file) + cls.__build_image(docker_file=build_file, cwd=repo_root_dir) + + @classmethod + def setUp(cls) -> None: + cls.__remove_generated_files() + os.makedirs(f"{golden_dir}", exist_ok=True) + + def test_entry_point_running_in_container(self): + api_definitions_path = self.__copy_api_definition(googleapis_commitish) + config_files = self.__get_config_files(config_dir) + for repo, config_file in config_files: + config = from_yaml(config_file) + repo_location = f"{output_dir}/{repo}" + config_location = f"{golden_dir}/../{repo}" + # 1. pull repository + repo_dest = self.__pull_repo_to( + Path(repo_location), repo, commitish_map[repo] + ) + # 2. prepare golden files + library_names = self.__get_library_names_from_config(config) + self.__prepare_golden_files( + config=config, library_names=library_names, repo_dest=repo_dest + ) + # 3. run entry_point.py in docker container + self.__run_entry_point_in_docker_container( + repo_location=repo_location, + config_location=config_location, + baseline_config=baseline_config_name, + current_config=current_config_name, + api_definition=api_definitions_path, + ) + # 4. generate pr description + # noinspection PyTypeChecker + result = CliRunner().invoke( + generate_pr_description, + [ + f"--baseline-generation-config-path={config_location}/{baseline_config_name}", + f"--current-generation-config-path={config_location}/{current_config_name}", + f"--repository-path={repo_location}", + ], + ) + self.assertEqual(0, result.exit_code) + # 5. compare generation result with golden files + print( + "Generation finished successfully. " + "Will now compare differences between generated and existing " + "libraries" + ) + for library_name in library_names: + actual_library = ( + f"{repo_dest}/{library_name}" if config.is_monorepo() else repo_dest + ) + print("*" * 50) + print(f"Checking for differences in '{library_name}'.") + print(f" The expected library is in {golden_dir}/{library_name}.") + print(f" The actual library is in {actual_library}. ") + compare_result = dircmp( + f"{golden_dir}/{library_name}", + actual_library, + ignore=[".repo-metadata.json"], + ) + diff_files = [] + golden_only = [] + generated_only = [] + # compare source code + self.__recursive_diff_files( + compare_result, diff_files, golden_only, generated_only + ) + + # print all found differences for inspection + print_file = lambda f: print(f" - {f}") + if len(diff_files) > 0: + print(" Some files (found in both folders) are differing:") + for diff_file in diff_files: + print(f"Difference in {diff_file}:") + with open( + f"{golden_dir}/{library_name}/{diff_file}" + ) as expected_file: + with open(f"{actual_library}/{diff_file}") as actual_file: + [ + print(line) + for line in difflib.unified_diff( + expected_file.readlines(), + actual_file.readlines(), + ) + ] + if len(golden_only) > 0: + print(" There were files found only in the golden dir:") + [print_file(f) for f in golden_only] + if len(generated_only) > 0: + print(" There were files found only in the generated dir:") + [print_file(f) for f in generated_only] + + self.assertTrue(len(golden_only) == 0) + self.assertTrue(len(generated_only) == 0) + self.assertTrue(len(diff_files) == 0) + + print(f" No differences found in {library_name}") + # compare .repo-metadata.json + self.assertTrue( + self.__compare_json_files( + f"{golden_dir}/{library_name}/.repo-metadata.json", + f"{actual_library}/.repo-metadata.json", + ), + msg=f" The generated {library_name}/.repo-metadata.json is different from golden.", + ) + print(" .repo-metadata.json comparison succeed.") + + if not config.is_monorepo(): + continue + + # compare gapic-libraries-bom/pom.xml and pom.xml + self.assertFalse( + compare_xml( + f"{golden_dir}/gapic-libraries-bom/pom.xml", + f"{repo_dest}/gapic-libraries-bom/pom.xml", + False, + ) + ) + print(" gapic-libraries-bom/pom.xml comparison succeed.") + self.assertFalse( + compare_xml( + f"{golden_dir}/pom.xml", + f"{repo_dest}/pom.xml", + False, + ) + ) + print(" pom.xml comparison succeed.") + # compare PR description + description_file = f"{output_dir}/{repo}/pr_description.txt" + self.assertTrue( + cmp( + f"{config_dir}/{repo}/pr-description-golden.txt", + f"{description_file}", + ), + "The generated PR description does not match the expected golden file", + ) + print(" PR description comparison succeed.") + self.__remove_generated_files() + shutil.rmtree(api_definitions_path) + + @classmethod + def __copy_api_definition(cls, committish: str) -> str: + repo_dest = cls.__pull_repo_to( + dest=tempfile.mkdtemp(), repo="googleapis", committish=committish + ) + api_temp_dir = tempfile.mkdtemp() + print(f"Copying api definition to {api_temp_dir}...") + shutil.copytree( + f"{repo_dest}/google", f"{api_temp_dir}/google", dirs_exist_ok=True + ) + shutil.copytree( + f"{repo_dest}/grafeas", f"{api_temp_dir}/grafeas", dirs_exist_ok=True + ) + shutil.rmtree(repo_dest) + return api_temp_dir + + @classmethod + def __build_image(cls, docker_file: str, cwd: str): + # we build the docker image without removing intermediate containers, so + # we can re-test more quickly + subprocess.check_call( + ["docker", "build", "-f", docker_file, "-t", image_tag, "."], + cwd=cwd, + ) + + @classmethod + def __download_generator_jar(cls, coordinates_file: str) -> None: + """ + Downloads the jar at the version specified in the + coordinates file + :param coordinates_file: path to the file containing the coordinates + """ + with open(coordinates_file, "r") as coordinates_file_handle: + # make this var available in the function scope + # nonlocal coordinates + coordinates = coordinates_file_handle.read() + # download the jar + subprocess.check_call( + [ + "mvn", + "dependency:copy", + f"-Dartifact={coordinates}", + f"-DoutputDirectory={config_dir}", + ] + ) + + # compute the filename of the downloaded jar + split_coordinates = coordinates.split(":") + artifact_id = split_coordinates[1] + version = split_coordinates[2] + jar_filename = f"{artifact_id}-{version}.jar" + + # rename the jar to its well-known filename defined at the top of this + # script file + source_jar_path = os.path.join(config_dir, jar_filename) + destination_jar_path = os.path.join( + config_dir, WELL_KNOWN_GENERATOR_JAR_FILENAME + ) + shutil.move(source_jar_path, destination_jar_path) + + @classmethod + def __remove_generated_files(cls): + shutil.rmtree(f"{output_dir}", ignore_errors=True) + if os.path.isdir(f"{golden_dir}"): + shutil.rmtree(f"{golden_dir}") + + @classmethod + def __pull_repo_to(cls, dest: Path, repo: str, committish: str) -> str: + shutil.rmtree(dest, ignore_errors=True) + repo_url = f"{repo_prefix}/{repo}" + print(f"Cloning repository {repo_url}") + repo = Repo.clone_from(repo_url, dest) + repo.git.checkout(committish) + return str(dest) + + @classmethod + def __get_library_names_from_config(cls, config: GenerationConfig) -> list[str]: + library_names = [] + for library in config.libraries: + library_names.append(f"java-{library.get_library_name()}") + + return library_names + + @classmethod + def __prepare_golden_files( + cls, config: GenerationConfig, library_names: list[str], repo_dest: str + ): + for library_name in library_names: + if config.is_monorepo(): + copy_tree(f"{repo_dest}/{library_name}", f"{golden_dir}/{library_name}") + copy_tree( + f"{repo_dest}/gapic-libraries-bom", + f"{golden_dir}/gapic-libraries-bom", + ) + copy_file(f"{repo_dest}/pom.xml", golden_dir) + else: + copy_tree(f"{repo_dest}", f"{golden_dir}/{library_name}") + + @classmethod + def __run_entry_point_in_docker_container( + cls, + repo_location: str, + config_location: str, + baseline_config: str, + current_config: str, + api_definition: str, + ): + # we use the calling user to prevent the mapped volumes from changing + # owners + user_id = shell_call("id -u") + group_id = shell_call("id -g") + subprocess.check_call( + [ + "docker", + "run", + "-u", + f"{user_id}:{group_id}", + "--rm", + "-v", + f"{repo_location}:/workspace/repo", + "-v", + f"{config_location}:/workspace/config", + "-v", + f"{api_definition}:/workspace/api", + "-v", + f"{config_dir}/{WELL_KNOWN_GENERATOR_JAR_FILENAME}:/home/.library_generation/{WELL_KNOWN_GENERATOR_JAR_FILENAME}", + "-w", + "/workspace/repo", + image_tag, + f"--baseline-generation-config-path=/workspace/config/{baseline_config}", + f"--current-generation-config-path=/workspace/config/{current_config}", + f"--api-definitions-path=/workspace/api", + ], + ) + + @classmethod + def __get_config_files(cls, path: str) -> list[tuple[str, str]]: + config_files = [] + for sub_dir in Path(path).resolve().iterdir(): + if sub_dir.is_file(): + continue + repo = sub_dir.name + if repo in ["golden", "java-bigtable"]: + continue + config = f"{sub_dir}/{current_config_name}" + config_files.append((repo, config)) + return config_files + + @classmethod + def __compare_json_files(cls, expected: str, actual: str) -> bool: + return cls.__load_json_to_sorted_list( + expected + ) == cls.__load_json_to_sorted_list(actual) + + @classmethod + def __load_json_to_sorted_list(cls, path: str) -> list[tuple]: + with open(path) as f: + data = json.load(f) + res = [(key, value) for key, value in data.items()] + + return sorted(res, key=lambda x: x[0]) + + @classmethod + def __recursive_diff_files( + cls, + dcmp: dircmp, + diff_files: list[str], + left_only: list[str], + right_only: list[str], + dirname: str = "", + ): + """ + Recursively compares two subdirectories. The found differences are + passed to three expected list references. + """ + append_dirname = lambda d: dirname + d + diff_files.extend(map(append_dirname, dcmp.diff_files)) + left_only.extend(map(append_dirname, dcmp.left_only)) + right_only.extend(map(append_dirname, dcmp.right_only)) + for sub_dirname, sub_dcmp in dcmp.subdirs.items(): + cls.__recursive_diff_files( + sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" + ) diff --git a/hermetic_build/library_generation/tests/model/__init__.py b/hermetic_build/library_generation/tests/model/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/model/config_change_unit_tests.py b/hermetic_build/library_generation/tests/model/config_change_unit_tests.py new file mode 100644 index 0000000000..6e0a088e75 --- /dev/null +++ b/hermetic_build/library_generation/tests/model/config_change_unit_tests.py @@ -0,0 +1,314 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.model.config_change import ChangeType +from library_generation.model.config_change import ConfigChange +from library_generation.model.config_change import LibraryChange +from common.model.gapic_config import GapicConfig +from common.model.generation_config import GenerationConfig +from common.model.library_config import LibraryConfig + + +class ConfigChangeTest(unittest.TestCase): + def test_get_changed_libraries_with_repo_level_change_returns_all_libraries_changed( + self, + ): + config_change = ConfigChange( + change_to_libraries={ + ChangeType.REPO_LEVEL_CHANGE: [], + # add a library level change to verify this type of change has + # no impact on the result. + ChangeType.LIBRARY_LEVEL_CHANGE: [ + LibraryChange( + changed_param="test", + current_value="test", + library_name="test-library", + ) + ], + }, + baseline_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), + ) + self.assertEqual( + ConfigChange.ALL_LIBRARIES_CHANGED, + config_change.get_changed_libraries(), + ) + + def test_get_changed_libraries_with_library_level_change_returns_list(self): + config_change = ConfigChange( + change_to_libraries={ + ChangeType.LIBRARY_LEVEL_CHANGE: [ + LibraryChange( + changed_param="test", + current_value="test", + library_name="a-library", + ), + LibraryChange( + changed_param="test", + current_value="test", + library_name="another-library", + ), + ], + ChangeType.LIBRARIES_ADDITION: [ + LibraryChange( + changed_param="test", + current_value="test", + library_name="new-library", + ), + ], + ChangeType.GAPIC_ADDITION: [ + LibraryChange( + changed_param="test", + current_value="test", + library_name="library-with-new-version", + ), + ], + }, + baseline_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), + ) + self.assertEqual( + ["a-library", "another-library", "library-with-new-version", "new-library"], + config_change.get_changed_libraries(), + ) + + def test_get_changed_libraries_with_duplicated_library_name_returns_unique_name( + self, + ): + config_change = ConfigChange( + change_to_libraries={ + ChangeType.LIBRARY_LEVEL_CHANGE: [ + LibraryChange( + changed_param="a-param", + current_value="new_test", + library_name="a-library", + ), + LibraryChange( + changed_param="another-param", + current_value="new_value", + library_name="a-library", + ), + ], + }, + baseline_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), + ) + self.assertEqual( + ["a-library"], + config_change.get_changed_libraries(), + ) + + def test_get_changed_libraries_with_mix_changes_returns_list(self): + baseline_commit = "277145d108819fa30fbed3a7cbbb50f91eb6155e" + latest_commit = "8984ddb508dea0e673b724c58338e810b1d8aee3" + config_change = ConfigChange( + change_to_libraries={ + ChangeType.GOOGLEAPIS_COMMIT: [], + ChangeType.LIBRARY_LEVEL_CHANGE: [ + LibraryChange( + changed_param="test", + current_value="test", + library_name="a-library", + ) + ], + ChangeType.LIBRARIES_ADDITION: [ + LibraryChange( + changed_param="test", + current_value="test", + library_name="new-library", + ), + ], + }, + baseline_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=baseline_commit + ), + current_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=latest_commit, + libraries=[ + ConfigChangeTest.__get_a_library_config( + library_name="gke-backup", + gapic_configs=[ + GapicConfig(proto_path="google/cloud/gkebackup/v1") + ], + ), + ], + ), + ) + + self.assertEqual( + ["a-library", "gke-backup", "new-library"], + sorted(config_change.get_changed_libraries()), + ) + + def test_get_qualified_commits_success(self): + baseline_commit = "277145d108819fa30fbed3a7cbbb50f91eb6155e" + latest_commit = "8984ddb508dea0e673b724c58338e810b1d8aee3" + gke_backup_commit = "b8691edb3f1d3c1583aa9cd89240eb359eebe9c7" + aiplatform_commit = "b82095baef02e525bee7bb1c48911c33b66acdf0" + network_management_commit = "efad09c9f0d46ae0786d810a88024363e06c6ca3" + config_change = ConfigChange( + change_to_libraries={}, + baseline_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=baseline_commit + ), + current_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=latest_commit, + libraries=[ + ConfigChangeTest.__get_a_library_config( + library_name="gke-backup", + gapic_configs=[ + GapicConfig(proto_path="google/cloud/gkebackup/v1") + ], + ), + ConfigChangeTest.__get_a_library_config( + library_name="aiplatform", + gapic_configs=[ + GapicConfig(proto_path="google/cloud/aiplatform/v1beta1") + ], + ), + ConfigChangeTest.__get_a_library_config( + library_name="network-management", + gapic_configs=[ + GapicConfig(proto_path="google/cloud/networkmanagement/v1"), + GapicConfig( + proto_path="google/cloud/networkmanagement/v1beta1" + ), + ], + ), + ], + ), + ) + qualified_commits = config_change.get_qualified_commits() + self.assertEqual(3, len(qualified_commits)) + self.assertEqual({"gke-backup"}, qualified_commits[0].libraries) + self.assertEqual( + gke_backup_commit, + qualified_commits[0].commit.hexsha, + ) + self.assertEqual({"aiplatform"}, qualified_commits[1].libraries) + self.assertEqual( + aiplatform_commit, + qualified_commits[1].commit.hexsha, + ) + self.assertEqual({"network-management"}, qualified_commits[2].libraries) + self.assertEqual( + network_management_commit, + qualified_commits[2].commit.hexsha, + ) + + def test_get_qualified_commits_rest_numeric_enum_change_returns_a_qualified_commit( + self, + ): + baseline_commit = "45694d2bad602c52170096072d325aa644d550e5" + latest_commit = "758f0d1217d9c7fe398aa5efb1057ce4b6409e55" + config_change = ConfigChange( + change_to_libraries={}, + baseline_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=baseline_commit + ), + current_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=latest_commit, + libraries=[ + ConfigChangeTest.__get_a_library_config( + library_name="container", + gapic_configs=[GapicConfig(proto_path="google/container/v1")], + ) + ], + ), + ) + # one commit between latest_commit and baseline_commit which only + # changed BUILD.bazel. + # this commit changed `rest_numeric_enums`. + self.assertTrue(len(config_change.get_qualified_commits()) == 1) + + def test_get_qualified_commits_irrelevant_build_field_change_returns_empty_list( + self, + ): + baseline_commit = "bdda0174f68a738518ec311e05e6fd9bbe19cd78" + latest_commit = "c9a5050ef225b0011603e1109cf53ab1de0a8e53" + config_change = ConfigChange( + change_to_libraries={}, + baseline_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=baseline_commit + ), + current_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=latest_commit, + libraries=[ + ConfigChangeTest.__get_a_library_config( + library_name="chat", + gapic_configs=[GapicConfig(proto_path="google/chat/v1")], + ) + ], + ), + ) + # one commit between latest_commit and baseline_commit which only + # changed BUILD.bazel. + # this commit didn't change fields used in library generation. + self.assertTrue(len(config_change.get_qualified_commits()) == 0) + + def test_get_qualified_commits_add_build_file_returns_a_qualified_commit(self): + baseline_commit = "d007ca1b3cc820651530d44d5388533047ae1414" + latest_commit = "05d889e7dfe087fc2ddc9de9579f01d4e1c2f35e" + config_change = ConfigChange( + change_to_libraries={}, + baseline_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=baseline_commit + ), + current_config=ConfigChangeTest.__get_a_gen_config( + googleapis_commitish=latest_commit, + libraries=[ + ConfigChangeTest.__get_a_library_config( + library_name="cloudcontrolspartner", + gapic_configs=[ + GapicConfig( + proto_path="google/cloud/cloudcontrolspartner/v1" + ) + ], + ) + ], + ), + ) + # one commit between latest_commit and baseline_commit which added + # google/cloud/cloudcontrolspartner/v1. + self.assertTrue(len(config_change.get_qualified_commits()) == 1) + + @staticmethod + def __get_a_gen_config( + googleapis_commitish="", libraries: list[LibraryConfig] = None + ) -> GenerationConfig: + if libraries is None: + libraries = [] + return GenerationConfig( + gapic_generator_version="", + googleapis_commitish=googleapis_commitish, + grpc_version="", + protoc_version="", + libraries=libraries, + ) + + @staticmethod + def __get_a_library_config( + library_name: str, gapic_configs: list[GapicConfig] = None + ) -> LibraryConfig: + if gapic_configs is None: + gapic_configs = [] + return LibraryConfig( + api_shortname="existing_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=gapic_configs, + library_name=library_name, + ) diff --git a/hermetic_build/library_generation/tests/model/repo_config_unit_tests.py b/hermetic_build/library_generation/tests/model/repo_config_unit_tests.py new file mode 100644 index 0000000000..12d28fe254 --- /dev/null +++ b/hermetic_build/library_generation/tests/model/repo_config_unit_tests.py @@ -0,0 +1,56 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest + +from library_generation.model.repo_config import RepoConfig + +script_dir = os.path.dirname(os.path.realpath(__file__)) +versions_file = os.path.join(script_dir, "..", "resources", "misc", "versions.txt") + + +class RepoConfigTest(unittest.TestCase): + def test_get_library_versions_with_existing_library(self): + repo_config = RepoConfig( + output_folder="test", libraries=dict(), versions_file=versions_file + ) + self.assertEqual( + "2.25.0", + repo_config.get_library_version("gapic-generator-java"), + ) + self.assertEqual( + "2.16.0", + repo_config.get_library_version("api-common"), + ) + self.assertEqual( + "2.33.0", + repo_config.get_library_version("gax"), + ) + self.assertEqual( + "2.34.0", + repo_config.get_library_version("gax-grpc"), + ) + self.assertEqual( + "0.118.0", + repo_config.get_library_version("gax-httpjson"), + ) + + def test_get_library_versions_with_new_library(self): + repo_config = RepoConfig( + output_folder="test", libraries=dict(), versions_file=versions_file + ) + self.assertEqual( + "0.0.0", + repo_config.get_library_version("example-artifact"), + ) diff --git a/hermetic_build/library_generation/tests/owlbot/__init__.py b/hermetic_build/library_generation/tests/owlbot/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py b/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py new file mode 100644 index 0000000000..2e66454827 --- /dev/null +++ b/hermetic_build/library_generation/tests/owlbot/fix_poms_unit_tests.py @@ -0,0 +1,47 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import shutil +import unittest +from library_generation.owlbot.src.fix_poms import main +from library_generation.tests.compare_poms import compare_pom_in_subdir + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources", "test-owlbot") + + +class FixPomsTest(unittest.TestCase): + def test_update_poms_group_id_does_not_start_with_google_correctly(self): + ad_manager_resource = os.path.join(resources_dir, "java-admanager") + versions_file = os.path.join(ad_manager_resource, "versions.txt") + os.chdir(ad_manager_resource) + sub_dirs = ["ad-manager", "ad-manager-bom", "proto-ad-manager-v1", "."] + for sub_dir in sub_dirs: + self.__copy__golden(ad_manager_resource, sub_dir) + main(versions_file, "true") + for sub_dir in sub_dirs: + self.assertFalse(compare_pom_in_subdir(ad_manager_resource, sub_dir)) + for sub_dir in sub_dirs: + self.__remove_file_in_subdir(ad_manager_resource, sub_dir) + + @classmethod + def __copy__golden(cls, base_dir: str, subdir: str): + golden = os.path.join(base_dir, subdir, "pom-golden.xml") + pom = os.path.join(base_dir, subdir, "pom.xml") + shutil.copyfile(golden, pom) + + @classmethod + def __remove_file_in_subdir(cls, base_dir: str, subdir: str): + pom = os.path.join(base_dir, subdir, "pom.xml") + os.unlink(pom) diff --git a/hermetic_build/library_generation/tests/owlbot/java_unit_tests.py b/hermetic_build/library_generation/tests/owlbot/java_unit_tests.py new file mode 100644 index 0000000000..e59486f6ad --- /dev/null +++ b/hermetic_build/library_generation/tests/owlbot/java_unit_tests.py @@ -0,0 +1,310 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tempfile +import unittest +import xml.etree.ElementTree as elementTree +from unittest import mock + +import yaml +from pathlib import Path +import requests_mock +from synthtool.languages import java +from library_generation.tests.owlbot import util + +TEST_OWLBOT = Path(__file__).parent.parent / "resources" / "test-owlbot" +FIXTURES = Path(__file__).parent.parent / "resources" / "test-owlbot" / "fixtures" +TEMPLATES_PATH = Path(__file__).parent.parent.parent / "owlbot" / "templates" + +SAMPLE_METADATA = """ + + com.google.cloud + libraries-bom + + 3.3.0 + 3.3.0 + + 1.0.0 + 1.1.0 + 1.1.1 + 1.2.0 + 2.0.0 + 2.1.0 + 2.2.0 + 2.2.1 + 2.3.0 + 2.4.0 + 2.5.0 + 2.6.0 + 2.7.0 + 2.7.1 + 2.8.0 + 2.9.0 + 3.0.0 + 3.1.0 + 3.1.1 + 3.2.0 + 3.3.0 + + 20191218182827 + + +""" + + +class JavaUnitTests(unittest.TestCase): + + def test_version_from_maven_metadata(self): + self.assertEqual("3.3.0", java.version_from_maven_metadata(SAMPLE_METADATA)) + + def test_latest_maven_version(self): + with requests_mock.Mocker() as m: + m.get( + "https://repo1.maven.org/maven2/com/google/cloud/libraries-bom/maven-metadata.xml", + text=SAMPLE_METADATA, + ) + self.assertEqual( + "3.3.0", + java.latest_maven_version( + group_id="com.google.cloud", artifact_id="libraries-bom" + ), + ) + + @mock.patch.dict(os.environ, {"SYNTHTOOL_TEMPLATES": f"{TEMPLATES_PATH}"}) + def test_working_common_templates(self): + def assert_valid_xml(file): + try: + elementTree.parse(file) + except elementTree.ParseError: + self.fail(f"unable to parse XML: {file}") + + def assert_valid_yaml(file): + with open(file, "r") as stream: + try: + yaml.safe_load(stream) + except yaml.YAMLError: + self.fail(f"unable to parse YAML: {file}") + + with util.copied_fixtures_dir( + FIXTURES / "java_templates" / "standard" + ) as workdir: + # generate the common templates + java.common_templates(template_path=TEMPLATES_PATH) + self.assertTrue(os.path.isfile("renovate.json")) + + # lint xml, yaml files + # use os.walk because glob ignores hidden directories + for dirpath, _, filenames in os.walk(workdir): + for file in filenames: + (_, ext) = os.path.splitext(file) + if ext == ".xml": + assert_valid_xml(os.path.join(dirpath, file)) + elif ext == ".yaml" or ext == ".yml": + assert_valid_yaml(os.path.join(dirpath, file)) + + def test_remove_method(self): + with tempfile.TemporaryDirectory() as tempdir: + cwd = os.getcwd() + os.chdir(TEST_OWLBOT) + shutil.copyfile("testdata/SampleClass.java", tempdir + "/SampleClass.java") + + java.remove_method( + tempdir + "/SampleClass.java", "public static void foo()" + ) + java.remove_method(tempdir + "/SampleClass.java", "public void asdf()") + self.assert_matches_golden( + "testdata/SampleClassGolden.java", tempdir + "/SampleClass.java" + ) + os.chdir(cwd) + + def test_copy_and_rename_method(self): + with tempfile.TemporaryDirectory() as tempdir: + cwd = os.getcwd() + os.chdir(TEST_OWLBOT) + shutil.copyfile("testdata/SampleClass.java", tempdir + "/SampleClass.java") + + java.copy_and_rename_method( + tempdir + "/SampleClass.java", + "public static void foo()", + "foo", + "foobar", + ) + java.copy_and_rename_method( + tempdir + "/SampleClass.java", "public void asdf()", "asdf", "xyz" + ) + self.assert_matches_golden( + "testdata/SampleCopyMethodGolden.java", + tempdir + "/SampleClass.java", + ) + os.chdir(cwd) + + def test_deprecate_method(self): + with tempfile.TemporaryDirectory() as tempdir: + cwd = os.getcwd() + os.chdir(TEST_OWLBOT) + shutil.copyfile( + "testdata/SampleDeprecateClass.java", + tempdir + "/SampleDeprecateClass.java", + ) + deprecation_warning = """This method will be removed in the next major version.\nUse {{@link #{new_method}()}} instead""" + additional_comment = ( + """{new_method} has the same functionality as foobar.""" + ) + java.deprecate_method( + tempdir + "/SampleDeprecateClass.java", + "public void foo(String bar)", + deprecation_warning.format(new_method="sample"), + ) + + # adding a comment when a javadoc and annotation already exists + java.deprecate_method( + tempdir + "/SampleDeprecateClass.java", + "public void bar(String bar)", + deprecation_warning.format(new_method="sample"), + ) + java.deprecate_method( + tempdir + "/SampleDeprecateClass.java", + "public void cat(String bar)", + additional_comment.format(new_method="sample"), + ) + + self.assert_matches_golden( + "testdata/SampleDeprecateMethodGolden.java", + tempdir + "/SampleDeprecateClass.java", + ) + os.chdir(cwd) + + def test_fix_proto_license(self): + with tempfile.TemporaryDirectory() as tempdir: + cwd = os.getcwd() + os.chdir(TEST_OWLBOT) + temppath = Path(tempdir).resolve() + os.mkdir(temppath / "src") + shutil.copyfile( + "testdata/src/foo/FooProto.java", temppath / "src/FooProto.java" + ) + + java.fix_proto_headers(temppath) + self.assert_matches_golden( + "testdata/FooProtoGolden.java", temppath / "src/FooProto.java" + ) + os.chdir(cwd) + + def test_fix_proto_license_idempotent(self): + with tempfile.TemporaryDirectory() as tempdir: + cwd = os.getcwd() + os.chdir(TEST_OWLBOT) + temppath = Path(tempdir).resolve() + os.mkdir(temppath / "src") + shutil.copyfile( + "testdata/src/foo/FooProto.java", temppath / "src/FooProto.java" + ) + + # run the header fix twice + java.fix_proto_headers(temppath) + java.fix_proto_headers(temppath) + self.assert_matches_golden( + "testdata/FooProtoGolden.java", temppath / "src/FooProto.java" + ) + os.chdir(cwd) + + def test_fix_grpc_license(self): + with tempfile.TemporaryDirectory() as tempdir: + cwd = os.getcwd() + os.chdir(TEST_OWLBOT) + temppath = Path(tempdir).resolve() + os.mkdir(temppath / "src") + shutil.copyfile( + "testdata/src/foo/FooGrpc.java", temppath / "src/FooGrpc.java" + ) + + java.fix_grpc_headers(temppath) + self.assert_matches_golden( + "testdata/FooGrpcGolden.java", temppath / "src/FooGrpc.java" + ) + os.chdir(cwd) + + def test_fix_grpc_license_idempotent(self): + with tempfile.TemporaryDirectory() as tempdir: + cwd = os.getcwd() + os.chdir(TEST_OWLBOT) + temppath = Path(tempdir).resolve() + os.mkdir(temppath / "src") + shutil.copyfile( + "testdata/src/foo/FooGrpc.java", temppath / "src/FooGrpc.java" + ) + + # run the header fix twice + java.fix_grpc_headers(temppath) + java.fix_grpc_headers(temppath) + self.assert_matches_golden( + "testdata/FooGrpcGolden.java", temppath / "src/FooGrpc.java" + ) + os.chdir(cwd) + + @mock.patch.dict(os.environ, {"SYNTHTOOL_TEMPLATES": f"{TEMPLATES_PATH}"}) + def test_release_please_handle_releases(self): + with util.copied_fixtures_dir( + FIXTURES / "java_templates" / "release-please-update" + ): + # generate the common templates + java.common_templates(template_path=TEMPLATES_PATH) + + self.assertTrue(os.path.isfile(".github/release-please.yml")) + with open(".github/release-please.yml") as fp: + self.assertEqual( + fp.read(), + """branches: +- branch: 1.127.12-sp + bumpMinorPreMajor: true + handleGHRelease: true + releaseType: java-lts +bumpMinorPreMajor: true +handleGHRelease: true +releaseType: java-yoshi +""", + ) + + @mock.patch.dict( + os.environ, + { + "SYNTHTOOL_TEMPLATES": f"{TEMPLATES_PATH}", + "SYNTHTOOL_LIBRARY_VERSION": "1.2.3", + }, + ) + def test_render_readme_success(self): + golden_path = os.path.abspath(f"{TEST_OWLBOT}/testdata/README-golden.md") + with util.copied_fixtures_dir(FIXTURES / "java_templates" / "render-readme"): + # This method needs read .repo-metadata.json to render templates. + # The file is located in FIXTURES/java_templates/render-readme. + java.common_templates( + template_path=TEMPLATES_PATH, + ) + self.assertTrue(os.path.isfile("README.md")) + self.assert_matches_golden(golden_path, "README.md") + + def assert_matches_golden(self, expected, actual): + matching_lines = 0 + with open(actual, "rt") as fp: + with open(expected, "rt") as golden: + while True: + matching_lines += 1 + log_line = fp.readline() + expected = golden.readline() + self.assertEqual(repr(log_line), repr(expected)) + if not log_line: + break + assert matching_lines > 0 diff --git a/hermetic_build/library_generation/tests/owlbot/util.py b/hermetic_build/library_generation/tests/owlbot/util.py new file mode 100644 index 0000000000..da54846768 --- /dev/null +++ b/hermetic_build/library_generation/tests/owlbot/util.py @@ -0,0 +1,126 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import subprocess +import os +import pathlib +import shutil +import sys +import tempfile +import typing + + +def make_working_repo(working_dir: str, default_branch: str = "main"): + """Create a local repo that resembles a real repo. + + Specifically, it has a history of synth.py changes that actually change the + generated output. + """ + subprocess.check_call(["git", "init"], cwd=working_dir) + subprocess.check_call(["git", "checkout", "-b", default_branch], cwd=working_dir) + subprocess.check_call( + [ + "git", + "remote", + "add", + "origin", + "https://github.com/googleapis/nodejs-vision.git", + ] + ) + working_path = pathlib.Path(working_dir) + # The simplest possible synth.py. It generates one file with one line of text. + template = """import time +import json +import uuid +import subprocess + +# comment + +with open("generated.txt", "wt") as f: + f.write("a\\n") +metadata = { "updateTime": str(uuid.uuid4()), + "sources": [ + { + "git": { + "name": ".", + "remote": "https://github.com/googleapis/synthtool.git", + "sha": subprocess.run(["git", "log", "-1", "--pretty=%H"], universal_newlines=True, stdout=subprocess.PIPE).stdout.strip(), + } + }] +} +with open("synth.metadata", "wt") as f: + json.dump(metadata, f) +""" + # Write version a. + synth_py_path = working_path / "synth.py" + synth_py_path.write_text(template) + subprocess.check_call([sys.executable, str(synth_py_path)], cwd=working_dir) + subprocess.check_call(["git", "add", "-A"], cwd=working_dir) + subprocess.check_call(["git", "commit", "-m", "a"], cwd=working_dir) + + # Write version b. + text = template.replace('"a\\n"', '"b\\n"') + synth_py_path.write_text(text) + subprocess.check_call(["git", "commit", "-am", "b"], cwd=working_dir) + + # Write a version that has no effect on output. + text = text.replace("# comment", "# a different comment") + synth_py_path.write_text(text) + subprocess.check_call(["git", "commit", "-am", "comment"], cwd=working_dir) + + # Write version c. + text = text.replace('"b\\n"', '"c\\n"') + synth_py_path.write_text(text) + subprocess.check_call( + ["git", "commit", "-am", "c subject\n\nbody line 1\nbody line 2"], + cwd=working_dir, + ) + + return text + + +@contextlib.contextmanager +def chdir(path: typing.Union[pathlib.Path, str]): + """Context Manager to change the current working directory and restore the + previous working directory after completing the context. + + Args: + path (pathlib.Path, str) - The new current working directory. + Yields: + pathlib.Path - The new current working directory. + """ + old_cwd = os.getcwd() + os.chdir(str(path)) + try: + yield pathlib.Path(path) + finally: + os.chdir(old_cwd) + + +@contextlib.contextmanager +def copied_fixtures_dir(source: pathlib.Path): + """Context Manager to copy from a fixtures directory into a new temporary directory + and change the current working directory to that copy. Restores the original + current working directory after completing the context. + + Args: + source (pathlib.Path) - The directory to copy. + Yields: + pathlib.Path - The temporary directory with the copied contents. + """ + with tempfile.TemporaryDirectory() as tempdir: + workdir = shutil.copytree(source, pathlib.Path(tempdir) / "workspace") + with chdir(workdir): + yield workdir diff --git a/hermetic_build/library_generation/tests/resources/gapic_options/QueryServiceGrpc_copy.java b/hermetic_build/library_generation/tests/resources/gapic_options/QueryServiceGrpc_copy.java new file mode 100644 index 0000000000..cf2bf2b0ee --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/gapic_options/QueryServiceGrpc_copy.java @@ -0,0 +1,9 @@ +/** + * This file is only used in testing `remove_grpc_version` in utilities.sh. + */ +@javax.annotation.processing.Generated( + value = "by gRPC proto compiler 1.55.1", + comments = "Source: google/monitoring/v3/query_service.proto") +public final class QueryServiceGrpc_copy { + +} diff --git a/hermetic_build/library_generation/tests/resources/gapic_options/example.yaml b/hermetic_build/library_generation/tests/resources/gapic_options/example.yaml new file mode 100644 index 0000000000..7bd824910e --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/gapic_options/example.yaml @@ -0,0 +1 @@ +# this file is only used in testing `get_gapic_opts` in utilities.sh diff --git a/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.legacy.yaml b/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.legacy.yaml new file mode 100644 index 0000000000..7da8426f34 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.legacy.yaml @@ -0,0 +1 @@ +# this file is only used in testing `get_gapic_opts` in utilities.sh \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.yaml b/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.yaml new file mode 100644 index 0000000000..7bd824910e --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic.yaml @@ -0,0 +1 @@ +# this file is only used in testing `get_gapic_opts` in utilities.sh diff --git a/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic_legacy.yaml b/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic_legacy.yaml new file mode 100644 index 0000000000..7da8426f34 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/gapic_options/example_gapic_legacy.yaml @@ -0,0 +1 @@ +# this file is only used in testing `get_gapic_opts` in utilities.sh \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/gapic_options/example_grpc_service_config.json b/hermetic_build/library_generation/tests/resources/gapic_options/example_grpc_service_config.json new file mode 100644 index 0000000000..947c390835 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/gapic_options/example_grpc_service_config.json @@ -0,0 +1,3 @@ +{ + "comment": "this file is only used in testing `get_gapic_opts` in utilities.sh" +} diff --git a/hermetic_build/library_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml b/hermetic_build/library_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml new file mode 100644 index 0000000000..225b4620bf --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml @@ -0,0 +1,35 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +deep-remove-regex: +- "/java-bare-metal-solution/grpc-google-.*/src" +- "/java-bare-metal-solution/proto-google-.*/src" +- "/java-bare-metal-solution/google-.*/src" +- "/java-bare-metal-solution/samples/snippets/generated" + +deep-preserve-regex: +- "/java-bare-metal-solution/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" + +deep-copy-regex: +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/proto-google-.*/src" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/proto-google-cloud-bare-metal-solution-$1/src" +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/grpc-google-.*/src" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/grpc-google-cloud-bare-metal-solution-$1/src" +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/gapic-google-.*/src" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/google-cloud-bare-metal-solution/src" +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/samples/snippets/generated" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/samples/snippets/generated" + +api-name: baremetalsolution \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json b/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json new file mode 100644 index 0000000000..2cccd4b889 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json @@ -0,0 +1,16 @@ +{ + "api_shortname": "secretmanager", + "name_pretty": "Secret Management", + "product_documentation": "https://cloud.google.com/solutions/secrets-management/", + "api_description": "allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-secretmanager/latest/overview", + "release_level": "preview", + "transport": "http", + "language": "java", + "repo": "googleapis/google-cloud-java", + "repo_short": "java-secretmanager", + "distribution_name": "com.google.cloud:google-cloud-secretmanager", + "api_id": "secretmanager.googleapis.com", + "library_type": "GAPIC_AUTO", + "requires_billing": true +} \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json b/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json new file mode 100644 index 0000000000..0b3bbc2b55 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json @@ -0,0 +1,20 @@ +{ + "api_shortname": "baremetalsolution", + "name_pretty": "Bare Metal Solution", + "product_documentation": "https://cloud.google.com/bare-metal/docs", + "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", + "release_level": "preview", + "transport": "grpc", + "language": "java", + "repo": "googleapis/google-cloud-java", + "repo_short": "java-bare-metal-solution", + "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", + "api_id": "baremetalsolution.googleapis.com", + "library_type": "GAPIC_AUTO", + "requires_billing": true, + "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", + "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", + "recommended_package": "com.google.example", + "min_java_version": 8 +} \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json b/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json new file mode 100644 index 0000000000..b39c297a85 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json @@ -0,0 +1,21 @@ +{ + "api_shortname": "baremetalsolution", + "name_pretty": "Bare Metal Solution", + "product_documentation": "https://cloud.google.com/bare-metal/docs", + "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", + "release_level": "preview", + "transport": "grpc", + "language": "java", + "repo": "googleapis/java-bare-metal-solution", + "repo_short": "java-bare-metal-solution", + "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", + "api_id": "baremetalsolution.googleapis.com", + "library_type": "GAPIC_COMBO", + "requires_billing": true, + "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", + "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", + "extra_versioned_modules": "test-module", + "recommended_package": "com.google.example", + "min_java_version": 8 +} \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json b/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json new file mode 100644 index 0000000000..7730cd6987 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json @@ -0,0 +1,19 @@ +{ + "api_shortname": "baremetalsolution", + "name_pretty": "Bare Metal Solution", + "product_documentation": "https://cloud.google.com/bare-metal/docs", + "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", + "release_level": "preview", + "transport": "grpc", + "language": "java", + "repo": "googleapis/sdk-platform-java", + "repo_short": "java-bare-metal-solution", + "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", + "library_type": "OTHER", + "requires_billing": true, + "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", + "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", + "recommended_package": "com.google.example", + "min_java_version": 8 +} \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py b/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py new file mode 100644 index 0000000000..2ba11e6bba --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/goldens/owlbot-golden.py @@ -0,0 +1,36 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import synthtool as s +from synthtool.languages import java + + +for library in s.get_staging_dirs(): + # put any special-case replacements here + s.move(library) + +s.remove_staging_dirs() +java.common_templates(monorepo=True, excludes=[ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "java.header", + "license-checks.xml", + "renovate.json", + ".gitignore" +]) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/goldens/pr_description-golden.txt b/hermetic_build/library_generation/tests/resources/goldens/pr_description-golden.txt new file mode 100644 index 0000000000..1a0f874936 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/goldens/pr_description-golden.txt @@ -0,0 +1,17 @@ +This pull request is generated with proto changes between [googleapis/googleapis@3b6f144](https://github.com/googleapis/googleapis/commit/3b6f144d47b0a1d2115ab2445ec06e80cc324a44) (exclusive) and [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) (inclusive). + +BEGIN_COMMIT_OVERRIDE +BEGIN_NESTED_COMMIT +fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +chore: update the libraries_bom version to 2.3.4 +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +feat: Make Layout Parser generally available in V1 + +PiperOrigin-RevId: 638924855 + +Source Link: [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) +END_NESTED_COMMIT +END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt b/hermetic_build/library_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt new file mode 100644 index 0000000000..81f7c0c4d2 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt @@ -0,0 +1,10 @@ +This pull request is generated with proto changes between [googleapis/googleapis@3b6f144](https://github.com/googleapis/googleapis/commit/3b6f144d47b0a1d2115ab2445ec06e80cc324a44) (exclusive) and [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) (inclusive). + +BEGIN_COMMIT_OVERRIDE +BEGIN_NESTED_COMMIT +fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +chore: update the libraries_bom version to 2.3.4 +END_NESTED_COMMIT +END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt b/hermetic_build/library_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt new file mode 100644 index 0000000000..f89901f8c1 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt @@ -0,0 +1,5 @@ +BEGIN_COMMIT_OVERRIDE +BEGIN_NESTED_COMMIT +fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 +END_NESTED_COMMIT +END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/baseline_generation_config.yaml b/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/baseline_generation_config.yaml new file mode 100644 index 0000000000..9a27dd381d --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/baseline_generation_config.yaml @@ -0,0 +1,45 @@ +gapic_generator_version: 2.38.1 +protoc_version: 25.2 +googleapis_commitish: a17d4caf184b050d50cacf2b0d579ce72c31ce74 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." + release_level: "stable" + library_name: "apigee-connect" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 + + - api_shortname: alloydb + name_pretty: AlloyDB + product_documentation: https://cloud.google.com/alloydb/ + api_description: AlloyDB is a fully managed, PostgreSQL-compatible database service + with industry-leading performance, availability, and scale. + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/v1 + - proto_path: google/cloud/alloydb/v1alpha + - proto_path: google/cloud/alloydb/v1beta + + - api_shortname: alloydb + name_pretty: AlloyDB connectors + product_documentation: https://cloud.google.com/alloydb/docs + api_description: AlloyDB is a fully-managed, PostgreSQL-compatible database for + demanding transactional workloads. It provides enterprise-grade performance and + availability while maintaining 100% compatibility with open-source PostgreSQL. + library_name: alloydb-connectors + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/connectors/v1 + - proto_path: google/cloud/alloydb/connectors/v1alpha + - proto_path: google/cloud/alloydb/connectors/v1beta + + - api_shortname: cloudcontrolspartner + name_pretty: Cloud Controls Partner API + product_documentation: https://cloud.google.com/sovereign-controls-by-partners/docs/sovereign-partners + api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. + GAPICs: + - proto_path: google/cloud/cloudcontrolspartner/v1 + - proto_path: google/cloud/cloudcontrolspartner/v1beta diff --git a/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/current_generation_config.yaml b/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/current_generation_config.yaml new file mode 100644 index 0000000000..adc1d170d6 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/current_generation_config.yaml @@ -0,0 +1,45 @@ +gapic_generator_version: 2.38.1 +protoc_version: 25.2 +googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." + release_level: "stable" + library_name: "apigee-connect" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 + + - api_shortname: alloydb + name_pretty: AlloyDB + product_documentation: https://cloud.google.com/alloydb/ + api_description: AlloyDB is a fully managed, PostgreSQL-compatible database service + with industry-leading performance, availability, and scale. + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/v1 + - proto_path: google/cloud/alloydb/v1alpha + - proto_path: google/cloud/alloydb/v1beta + + - api_shortname: alloydb + name_pretty: AlloyDB connectors + product_documentation: https://cloud.google.com/alloydb/docs + api_description: AlloyDB is a fully-managed, PostgreSQL-compatible database for + demanding transactional workloads. It provides enterprise-grade performance and + availability while maintaining 100% compatibility with open-source PostgreSQL. + library_name: alloydb-connectors + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/connectors/v1 + - proto_path: google/cloud/alloydb/connectors/v1alpha + - proto_path: google/cloud/alloydb/connectors/v1beta + + - api_shortname: cloudcontrolspartner + name_pretty: Cloud Controls Partner API + product_documentation: https://cloud.google.com/sovereign-controls-by-partners/docs/sovereign-partners + api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. + GAPICs: + - proto_path: google/cloud/cloudcontrolspartner/v1 + - proto_path: google/cloud/cloudcontrolspartner/v1beta diff --git a/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/pr-description-golden.txt b/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/pr-description-golden.txt new file mode 100644 index 0000000000..cb855dcd52 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/integration/google-cloud-java/pr-description-golden.txt @@ -0,0 +1,54 @@ +This pull request is generated with proto changes between [googleapis/googleapis@a17d4ca](https://github.com/googleapis/googleapis/commit/a17d4caf184b050d50cacf2b0d579ce72c31ce74) (exclusive) and [googleapis/googleapis@4ce0ff6](https://github.com/googleapis/googleapis/commit/4ce0ff67a3d4509be641cbe47a35844ddc1268fc) (inclusive). + +BEGIN_COMMIT_OVERRIDE +BEGIN_NESTED_COMMIT +docs: [cloudcontrolspartner] update documentation URL + +PiperOrigin-RevId: 612723053 + +Source Link: [googleapis/googleapis@7659dd2](https://github.com/googleapis/googleapis/commit/7659dd2244563fd902b681bdcadbe5385b8d3462) +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +feat: [cloudcontrolspartner] added CloudControlsPartner API + +PiperOrigin-RevId: 612632640 + +Source Link: [googleapis/googleapis@05d889e](https://github.com/googleapis/googleapis/commit/05d889e7dfe087fc2ddc9de9579f01d4e1c2f35e) +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +feat: [alloydb] support for obtaining the public IP address of an Instance +feat: [alloydb] support for getting PSC DNS name from the GetConnectionInfo API +feat: [alloydb] add PSC cluster and instance configuration settings to enable/disable PSC and obtain the PSC endpoint name +feat: [alloydb] add new API to list the databases in a project and location +docs: [alloydb] clarified read pool config is for read pool type instances + +PiperOrigin-RevId: 610475013 + +Source Link: [googleapis/googleapis@aa16fda](https://github.com/googleapis/googleapis/commit/aa16fdad909bc33e2d4ff04cfde56a46d0e52b13) +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +feat: [alloydb] support for obtaining the public IP address of an Instance +feat: [alloydb] support for getting PSC DNS name from the GetConnectionInfo API + +PiperOrigin-RevId: 610415824 + +Source Link: [googleapis/googleapis@0733fdb](https://github.com/googleapis/googleapis/commit/0733fdb5f745192f9f3c95f8d08039286567cbcc) +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +docs: [cloudcontrolspartner] Updated comment for method `ListCustomers` in service `CloudControlsPartnerCore` +docs: [cloudcontrolspartner] Updated comment for field `location` in message `.google.cloud.cloudcontrolspartner.v1beta.Workload` +docs: [cloudcontrolspartner] Updated a comment for field `partner_project_id` in message `.google.cloud.cloudcontrolspartner.v1beta.Partner` +docs: [cloudcontrolspartner] Updated documentation URL + +PiperOrigin-RevId: 609026905 + +Source Link: [googleapis/googleapis@9e35c62](https://github.com/googleapis/googleapis/commit/9e35c620157d7b11cb5b2e5d0249c5caaee824f3) +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +feat: [cloudcontrolspartner] added CloudControlsPartner API + +PiperOrigin-RevId: 606720708 + +Source Link: [googleapis/googleapis@36dedd0](https://github.com/googleapis/googleapis/commit/36dedd0d9020c19d1c8259003c2fe9656ada7471) +END_NESTED_COMMIT +END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/integration/java-bigtable/generation_config.yaml b/hermetic_build/library_generation/tests/resources/integration/java-bigtable/generation_config.yaml new file mode 100644 index 0000000000..c6912c8125 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/integration/java-bigtable/generation_config.yaml @@ -0,0 +1,22 @@ +gapic_generator_version: 2.37.0 +protoc_version: 25.2 +googleapis_commitish: 9868a57470a969ffa1d21194a5c05d7a6e4e98cc +libraries: +- api_shortname: bigtable + name_pretty: Cloud Bigtable + api_description: "Java idiomatic client for Cloud Bigtable." + product_documentation: "https://cloud.google.com/bigtable" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-bigtable/latest/history" + issue_tracker: "https://issuetracker.google.com/savedsearches/559777" + release_level: "stable" + language: "java" + repo: "googleapis/java-bigtable" + repo_short: "java-bigtable" + distribution_name: "com.google.cloud:google-cloud-bigtable" + excluded_poms: "google-cloud-bigtable-bom" + extra_versioned_modules: "google-cloud-bigtable-emulator,google-cloud-bigtable-emulator-core" + codeowner_team: "@googleapis/api-bigtable @googleapis/api-bigtable-partners" + library_type: GAPIC_COMBO + GAPICs: + - proto_path: google/bigtable/admin/v2 + - proto_path: google/bigtable/v2 diff --git a/hermetic_build/library_generation/tests/resources/integration/java-bigtable/pr-description-golden.txt b/hermetic_build/library_generation/tests/resources/integration/java-bigtable/pr-description-golden.txt new file mode 100644 index 0000000000..2dcdbc7756 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/integration/java-bigtable/pr-description-golden.txt @@ -0,0 +1,18 @@ +This pull request is generated with proto changes between googleapis commit 679060c64136e85b52838f53cfe612ce51e60d1d (exclusive) and fc3043ebe12fb6bc1729c175e1526c859ce751d8 (inclusive). +Qualified commits are: +[googleapis/googleapis@fbcfef0](https://github.com/googleapis/googleapis/commit/fbcfef09510b842774530989889ed1584a8b5acb) +[googleapis/googleapis@63d2a60](https://github.com/googleapis/googleapis/commit/63d2a60056ad5b156c05c7fb13138fc886c3b739) +BEGIN_COMMIT_OVERRIDE +BEGIN_NESTED_COMMIT +fix: extend timeouts for deleting snapshots, backups and tables + +PiperOrigin-RevId: 605388988 + +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +chore: update retry settings for backup rpcs + +PiperOrigin-RevId: 605367937 + +END_NESTED_COMMIT +END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/integration/test_generator_coordinates b/hermetic_build/library_generation/tests/resources/integration/test_generator_coordinates new file mode 100644 index 0000000000..00af5f647f --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/integration/test_generator_coordinates @@ -0,0 +1 @@ +com.google.api:gapic-generator-java:2.38.1 \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_common_resources.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_common_resources.bazel new file mode 100644 index 0000000000..126ffdb7ca --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_common_resources.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + #"//google/cloud:common_resources_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_iam_policy.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_iam_policy.bazel new file mode 100644 index 0000000000..a9a2c1ca75 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_iam_policy.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + # "//google/iam/v1:iam_policy_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_locations.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_locations.bazel new file mode 100644 index 0000000000..8b96e3ab81 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_locations.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + # "//google/cloud/location:location_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_common_resources.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_common_resources.bazel new file mode 100644 index 0000000000..9b749e6ad5 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_common_resources.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + "//google/cloud:common_resources_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_gapic_yaml.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_gapic_yaml.bazel new file mode 100644 index 0000000000..b55f4550d8 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_gapic_yaml.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + gapic_yaml = "test_gapic_yaml.yaml", +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc.bazel new file mode 100644 index 0000000000..3e59a9bd35 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_transport_from_BUILD` in utilities.sh + +java_gapic_library( + transport = "grpc", +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc_rest.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc_rest.bazel new file mode 100644 index 0000000000..8a98e7e646 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc_rest.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_transport_from_BUILD` in utilities.sh + +java_gapic_library( + transport = "grpc+rest", +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_locations.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_locations.bazel new file mode 100644 index 0000000000..d0c971da7c --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_locations.bazel @@ -0,0 +1,6 @@ +proto_library_with_info( + deps = [ + "//google/cloud/location:location_proto", + "//google/iam/v1:iam_policy_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_policy.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_policy.bazel new file mode 100644 index 0000000000..af5d4a32f8 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_policy.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + "//google/iam/v1:iam_policy_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_empty.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_empty.bazel new file mode 100644 index 0000000000..b9d4cd56e0 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_empty.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh + +java_gapic_assembly_gradle_pkg( + +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_false.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_false.bazel new file mode 100644 index 0000000000..8e95c3d2a6 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_false.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh + +java_gapic_assembly_gradle_pkg( + include_samples = False, +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_true.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_true.bazel new file mode 100644 index 0000000000..bac72b678f --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_true.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh + +java_gapic_assembly_gradle_pkg( + include_samples = True, +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_locations.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_locations.bazel new file mode 100644 index 0000000000..29ee14fdba --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_locations.bazel @@ -0,0 +1,5 @@ +proto_library_with_info( + deps = [ + "//google/cloud/location:location_proto", + ] +) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_additional_protos.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_additional_protos.bazel new file mode 100644 index 0000000000..a22257cad4 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_additional_protos.bazel @@ -0,0 +1,4 @@ +proto_library_with_info( + deps = [ + ] +) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_gapic_yaml.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_gapic_yaml.bazel new file mode 100644 index 0000000000..1e9462aa30 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_gapic_yaml.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + gapic_yaml = None +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_config.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_config.bazel new file mode 100644 index 0000000000..dbde6de05c --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_config.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + grpc_service_config = None +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_yaml.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_yaml.bazel new file mode 100644 index 0000000000..05bae16d5d --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_yaml.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + service_yaml = None +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_proto_only.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_proto_only.bazel new file mode 100644 index 0000000000..26bcea6126 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_proto_only.bazel @@ -0,0 +1,16 @@ +java_gapic_assembly_gradle_pkg( + name = "google-api-java", + transport = "grpc+rest", + deps = [ + "annotations_proto", + "auth_proto", + "backend_proto", + "billing_proto", + "client_proto", + "config_change_proto", + "consumer_proto", + "context_proto", + "control_proto", + "distribution_proto", + ], +) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest.bazel new file mode 100644 index 0000000000..9dff694297 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_transport_from_BUILD` in utilities.sh + +java_gapic_library( + transport = "rest", +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel new file mode 100644 index 0000000000..992b91e52c --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh + +java_gapic_library( + +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel new file mode 100644 index 0000000000..a446c6b2a7 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh + +java_gapic_library( + rest_numeric_enums = False +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel new file mode 100644 index 0000000000..c4d7fefeb4 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel @@ -0,0 +1,5 @@ +# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh + +java_gapic_library( + rest_numeric_enums = True +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config.bazel new file mode 100644 index 0000000000..097d1bb6bd --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + grpc_service_config = "test_service_config.json" +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config_relative_target.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config_relative_target.bazel new file mode 100644 index 0000000000..ccd59af2fb --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config_relative_target.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + grpc_service_config = ":compute_grpc_service_config.json" +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml.bazel new file mode 100644 index 0000000000..f7e4c91f4e --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + service_yaml = "test_service_yaml.yaml" +) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel new file mode 100644 index 0000000000..ded899dff7 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel @@ -0,0 +1,3 @@ +java_gapic_library( + service_yaml = "//google/cloud/videointelligence:videointelligence_v1p3beta1.yaml", +) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/TESTWORKSPACE b/hermetic_build/library_generation/tests/resources/misc/TESTWORKSPACE new file mode 100644 index 0000000000..60b3036d9d --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/TESTWORKSPACE @@ -0,0 +1,133 @@ +# test workspace file obtained from sdk-platform-java + +workspace(name = "gapic_generator_java") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# gax-java and its transitive dependencies must be imported before +# gapic-generator-java dependencies to match the order in googleapis repository, +# which in its turn, prioritizes actual generated clients runtime dependencies +# over the generator dependencies. +local_repository( + name = "com_google_api_gax_java", + path = "gax-java", +) + +load("@com_google_api_gax_java//:repository_rules.bzl", "com_google_api_gax_java_properties") + +com_google_api_gax_java_properties( + name = "com_google_api_gax_java_properties", + file = "@com_google_api_gax_java//:dependencies.properties", +) + +load("@com_google_api_gax_java//:repositories.bzl", "com_google_api_gax_java_repositories") + +com_google_api_gax_java_repositories() + +_googleapis_commit = "7438480b2a1bc6371d748e974f7a3647f90c4e8d" + +http_archive( + name = "com_google_googleapis", + strip_prefix = "googleapis-%s" % _googleapis_commit, + urls = [ + "https://github.com/googleapis/googleapis/archive/%s.zip" % _googleapis_commit, + ], +) + +# protobuf +RULES_JVM_EXTERNAL_TAG = "4.5" + +RULES_JVM_EXTERNAL_SHA = "b17d7388feb9bfa7f2fa09031b32707df529f26c91ab9e5d909eb1676badd9a6" + +http_archive( + name = "rules_jvm_external", + sha256 = RULES_JVM_EXTERNAL_SHA, + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, + url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, +) + +load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") + +rules_jvm_external_deps() + +load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") + +rules_jvm_external_setup() + +load("@com_google_protobuf//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") +load("@rules_jvm_external//:defs.bzl", "maven_install") + +maven_install( + artifacts = PROTOBUF_MAVEN_ARTIFACTS, + repositories = ["https://repo.maven.apache.org/maven2/"], +) + +_gapic_generator_java_version = "2.25.1-SNAPSHOT" # {x-version-update:gapic-generator-java:current} + +maven_install( + artifacts = [ + "com.google.api:gapic-generator-java:" + _gapic_generator_java_version, + ], + fail_on_missing_checksum = False, + repositories = [ + "m2Local", + "https://repo.maven.apache.org/maven2/", + ], +) + +protobuf_deps() + +# Bazel rules. +_rules_gapic_version = "0.5.5" + +http_archive( + name = "rules_gapic", + strip_prefix = "rules_gapic-%s" % _rules_gapic_version, + urls = ["https://github.com/googleapis/rules_gapic/archive/v%s.tar.gz" % _rules_gapic_version], +) + +# Java dependencies. +load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") + +switched_rules_by_language( + name = "com_google_googleapis_imports", + gapic = True, + grpc = True, + java = True, +) + +load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories") + +grpc_java_repositories() + +_disco_to_proto3_converter_commit = "ce8d8732120cdfb5bf4847c3238b5be8acde87e3" + +http_archive( + name = "com_google_disco_to_proto3_converter", + strip_prefix = "disco-to-proto3-converter-%s" % _disco_to_proto3_converter_commit, + urls = ["https://github.com/googleapis/disco-to-proto3-converter/archive/%s.zip" % _disco_to_proto3_converter_commit], +) + +# Showcase +_showcase_version = "0.28.2" + +http_archive( + name = "com_google_gapic_showcase", + strip_prefix = "gapic-showcase-%s" % _showcase_version, + urls = [ + "https://github.com/googleapis/gapic-showcase/archive/refs/tags/v%s.zip" % _showcase_version, + ], +) + +http_archive( + name = "rules_pkg", + sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", + ], +) + +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") + +rules_pkg_dependencies() diff --git a/hermetic_build/library_generation/tests/resources/misc/versions.txt b/hermetic_build/library_generation/tests/resources/misc/versions.txt new file mode 100644 index 0000000000..e4258504e4 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/misc/versions.txt @@ -0,0 +1,9 @@ +# test versions.txt obtained from sdk-platform-java + +gapic-generator-java:2.25.0:2.25.1-SNAPSHOT +api-common:2.16.0:2.16.1-SNAPSHOT +gax:2.33.0:2.33.1-SNAPSHOT +gax-grpc:2.34.0:2.33.1-SNAPSHOT +gax-httpjson:0.118.0:0.118.1-SNAPSHOT +proto-google-common-protos:2.24.0:2.24.1-SNAPSHOT +grpc-google-common-protos:2.24.0:2.24.1-SNAPSHOT diff --git a/hermetic_build/library_generation/tests/resources/proto_path_library/proto-test-library/src/main/proto/google/cloud/test/v1/empty.proto b/hermetic_build/library_generation/tests/resources/proto_path_library/proto-test-library/src/main/proto/google/cloud/test/v1/empty.proto new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/resources/proto_path_library_multiple_protos/proto-1/fake.proto b/hermetic_build/library_generation/tests/resources/proto_path_library_multiple_protos/proto-1/fake.proto new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/resources/proto_path_library_multiple_protos/proto-2/fake.proto b/hermetic_build/library_generation/tests/resources/proto_path_library_multiple_protos/proto-2/fake.proto new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_api_description.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_api_description.yaml new file mode 100644 index 0000000000..79ff135067 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_api_description.yaml @@ -0,0 +1,5 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_api_shortname.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_api_shortname.yaml new file mode 100644 index 0000000000..ec8206be61 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_api_shortname.yaml @@ -0,0 +1,4 @@ +gapic_generator_version: 2.34.0 +libraries: + - GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_key.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_key.yaml new file mode 100644 index 0000000000..739a4d9239 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_key.yaml @@ -0,0 +1,3 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_value.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_value.yaml new file mode 100644 index 0000000000..ec49e4a669 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_value.yaml @@ -0,0 +1,4 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect + GAPICs: diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_googleapis.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_googleapis.yaml new file mode 100644 index 0000000000..e5a00ca4ee --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_googleapis.yaml @@ -0,0 +1,8 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + api_description: "allows the Apigee hybrid management" + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_libraries.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_libraries.yaml new file mode 100644 index 0000000000..dbbe2ea318 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_libraries.yaml @@ -0,0 +1 @@ +gapic_generator_version: 2.34.0 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_library_value.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_library_value.yaml new file mode 100644 index 0000000000..174a293000 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_library_value.yaml @@ -0,0 +1,2 @@ +gapic_generator_version: 2.34.0 +libraries: diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_name_pretty.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_name_pretty.yaml new file mode 100644 index 0000000000..f8612ad9ca --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_name_pretty.yaml @@ -0,0 +1,6 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect + api_description: "allows the Apigee hybrid management" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_product_docs.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_product_docs.yaml new file mode 100644 index 0000000000..e3921d2c0d --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_product_docs.yaml @@ -0,0 +1,7 @@ +gapic_generator_version: 2.34.0 +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + api_description: "allows the Apigee hybrid management" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_proto_path.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_proto_path.yaml new file mode 100644 index 0000000000..e37b0cef63 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_proto_path.yaml @@ -0,0 +1,4 @@ +gapic_generator_version: 2.34.0 +libraries: + - GAPICs: + - random_key: diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_temp_excludes.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_temp_excludes.yaml new file mode 100644 index 0000000000..0d1bb7deea --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/config_without_temp_excludes.yaml @@ -0,0 +1,10 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + api_description: "allows the Apigee hybrid management" + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/generation_config.yaml b/hermetic_build/library_generation/tests/resources/test-config/generation_config.yaml new file mode 100644 index 0000000000..168c8fd9a5 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/generation_config.yaml @@ -0,0 +1,24 @@ +gapic_generator_version: 2.34.0 +protoc_version: 25.2 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: "stable" + issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" + api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + codeowner_team: "@googleapis/analytics-dpe" + excluded_poms: proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1 + excluded_dependencies: google-iam-policy + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/generation_config_library_modified.yaml b/hermetic_build/library_generation/tests/resources/test-config/generation_config_library_modified.yaml new file mode 100644 index 0000000000..f9ae96693b --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/generation_config_library_modified.yaml @@ -0,0 +1,14 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml b/hermetic_build/library_generation/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml new file mode 100644 index 0000000000..c5613f4308 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml @@ -0,0 +1,51 @@ +gapic_generator_version: 2.34.0 +protoc_version: 25.2 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 +synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 +template_excludes: + - ".github/*" + - ".kokoro/*" + - "samples/*" + - "CODE_OF_CONDUCT.md" + - "CONTRIBUTING.md" + - "LICENSE" + - "SECURITY.md" + - "java.header" + - "license-checks.xml" + - "renovate.json" + - ".gitignore" +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: "stable" + issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" + api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + codeowner_team: "@googleapis/analytics-dpe" + excluded_poms: proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1 + excluded_dependencies: google-iam-policy + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + + - api_shortname: another-cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: "stable" + issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" + api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + GAPICs: + - proto_path: google/cloud/asset/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/monorepo_baseline.yaml b/hermetic_build/library_generation/tests/resources/test-config/monorepo_baseline.yaml new file mode 100644 index 0000000000..c2c4fd4a3b --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/monorepo_baseline.yaml @@ -0,0 +1,30 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: preview + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + - api_shortname: cloudbuild + name_pretty: Cloud Build + product_documentation: https://cloud.google.com/cloud-build/ + api_description: lets you build software quickly across all languages. Get complete + control over defining custom workflows for building, testing, and deploying across + multiple environments such as VMs, serverless, Kubernetes, or Firebase. + release_level: stable + distribution_name: com.google.cloud:google-cloud-build + issue_tracker: https://issuetracker.google.com/savedsearches/5226584 + GAPICs: + - proto_path: google/devtools/cloudbuild/v1 + - proto_path: google/devtools/cloudbuild/v2 diff --git a/hermetic_build/library_generation/tests/resources/test-config/monorepo_current.yaml b/hermetic_build/library_generation/tests/resources/test-config/monorepo_current.yaml new file mode 100644 index 0000000000..3ee2c8be2c --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/monorepo_current.yaml @@ -0,0 +1,30 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: stable + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + - api_shortname: cloudbuild + name_pretty: Cloud Build + product_documentation: https://cloud.google.com/cloud-build/ + api_description: lets you build software quickly across all languages. Get complete + control over defining custom workflows for building, testing, and deploying across + multiple environments such as VMs, serverless, Kubernetes, or Firebase. + release_level: stable + distribution_name: com.google.cloud:google-cloud-build + issue_tracker: https://issuetracker.google.com/savedsearches/5226584 + GAPICs: + - proto_path: google/devtools/cloudbuild/v1 + - proto_path: google/devtools/cloudbuild/v2 diff --git a/hermetic_build/library_generation/tests/resources/test-config/monorepo_with_common_protos.yaml b/hermetic_build/library_generation/tests/resources/test-config/monorepo_with_common_protos.yaml new file mode 100644 index 0000000000..6d4c94444a --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/monorepo_with_common_protos.yaml @@ -0,0 +1,43 @@ +googleapis_commitish: 6a474b31c53cc1797710206824a17b364a835d2d +gapic_generator_version: 2.34.0 +# the libraries are ordered with respect to library name, which is +# java-{library.library_name} or java-{library.api-shortname} when +# library.library_name is not defined. +libraries: +- api_shortname: common-protos + name_pretty: Common Protos + product_documentation: https://github.com/googleapis/api-common-protos + api_description: Protobuf classes for Google's common protos. + release_level: stable + client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history + distribution_name: com.google.api.grpc:proto-google-common-protos + excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" + excluded_poms: "proto-google-common-protos-bom,proto-google-common-protos" + library_type: OTHER + GAPICs: + - proto_path: google/api + - proto_path: google/apps/card/v1 + - proto_path: google/cloud + - proto_path: google/cloud/audit + - proto_path: google/cloud/location + - proto_path: google/geo/type + - proto_path: google/logging/type + - proto_path: google/longrunning + - proto_path: google/rpc + - proto_path: google/rpc/context + - proto_path: google/shopping/type + - proto_path: google/type +- api_shortname: iam + name_pretty: IAM + product_documentation: https://cloud.google.com/iam + api_description: Manages access control for Google Cloud Platform resources + release_level: stable + client_documentation: https://cloud.google.com/java/docs/reference/proto-google-iam-v1/latest/overview + distribution_name: com.google.api.grpc:proto-google-iam-v1 + excluded_dependencies: "grpc-google-iam-v1" + excluded_poms: "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1" + library_type: OTHER + GAPICs: + - proto_path: google/iam/v1 + - proto_path: google/iam/v2 + - proto_path: google/iam/v2beta \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test-config/monorepo_without_common_protos.yaml b/hermetic_build/library_generation/tests/resources/test-config/monorepo_without_common_protos.yaml new file mode 100644 index 0000000000..ca21ccfb01 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-config/monorepo_without_common_protos.yaml @@ -0,0 +1,33 @@ +gapic_generator_version: 2.34.0 +protoc_version: 25.2 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: "stable" + issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" + api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + - api_shortname: cloudbuild + name_pretty: Cloud Build + product_documentation: https://cloud.google.com/cloud-build/ + api_description: lets you build software quickly across all languages. Get complete + control over defining custom workflows for building, testing, and deploying across + multiple environments such as VMs, serverless, Kubernetes, or Firebase. + release_level: stable + distribution_name: com.google.cloud:google-cloud-build + issue_tracker: https://issuetracker.google.com/savedsearches/5226584 + GAPICs: + - proto_path: google/devtools/cloudbuild/v1 + - proto_path: google/devtools/cloudbuild/v2 diff --git a/hermetic_build/library_generation/tests/resources/test-monorepo/.github/.OwlBot.lock.yaml b/hermetic_build/library_generation/tests/resources/test-monorepo/.github/.OwlBot.lock.yaml new file mode 100644 index 0000000000..77200af4c9 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-monorepo/.github/.OwlBot.lock.yaml @@ -0,0 +1,17 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +docker: + image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest + digest: sha256:fb7584f6adb3847ac480ed49a4bfe1463965026b2919a1be270e3174f3ce1191 + # created: 2023-01-20T00:00:00.000000000Z diff --git a/hermetic_build/library_generation/tests/resources/test-monorepo/test-service/.repo-metadata.json b/hermetic_build/library_generation/tests/resources/test-monorepo/test-service/.repo-metadata.json new file mode 100644 index 0000000000..d5b5078213 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-monorepo/test-service/.repo-metadata.json @@ -0,0 +1,18 @@ +{ + "api_shortname": "cloudasset", + "name_pretty": "Cloud Asset Inventory", + "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview", + "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", + "release_level": "stable", + "transport": "grpc", + "requires_billing": true, + "language": "java", + "repo": "googleapis/google-cloud-java", + "repo_short": "java-asset", + "distribution_name": "com.google.cloud:google-cloud-asset", + "api_id": "cloudasset.googleapis.com", + "library_type": "GAPIC_AUTO" +} diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/release-please-update/.github/release-please.yml b/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/release-please-update/.github/release-please.yml new file mode 100644 index 0000000000..807174cce1 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/release-please-update/.github/release-please.yml @@ -0,0 +1,6 @@ +releaseType: java-yoshi +bumpMinorPreMajor: true +branches: +- releaseType: java-lts + bumpMinorPreMajor: true + branch: 1.127.12-sp \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/release-please-update/.repo-metadata.json b/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/release-please-update/.repo-metadata.json new file mode 100644 index 0000000000..840e69ca7a --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/release-please-update/.repo-metadata.json @@ -0,0 +1,18 @@ +{ + "api_shortname": "cloudasset", + "name_pretty": "Cloud Asset Inventory", + "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", + "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html", + "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", + "release_level": "stable", + "transport": "grpc", + "requires_billing": true, + "language": "java", + "repo": "googleapis/java-asset", + "repo_short": "java-asset", + "distribution_name": "com.google.cloud:google-cloud-asset", + "library_type": "GAPIC_AUTO", + "api_id": "cloudasset.googleapis.com" +} \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/render-readme/.repo-metadata.json b/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/render-readme/.repo-metadata.json new file mode 100644 index 0000000000..840e69ca7a --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/render-readme/.repo-metadata.json @@ -0,0 +1,18 @@ +{ + "api_shortname": "cloudasset", + "name_pretty": "Cloud Asset Inventory", + "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", + "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html", + "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", + "release_level": "stable", + "transport": "grpc", + "requires_billing": true, + "language": "java", + "repo": "googleapis/java-asset", + "repo_short": "java-asset", + "distribution_name": "com.google.cloud:google-cloud-asset", + "library_type": "GAPIC_AUTO", + "api_id": "cloudasset.googleapis.com" +} \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/standard/.repo-metadata.json b/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/standard/.repo-metadata.json new file mode 100644 index 0000000000..840e69ca7a --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/fixtures/java_templates/standard/.repo-metadata.json @@ -0,0 +1,18 @@ +{ + "api_shortname": "cloudasset", + "name_pretty": "Cloud Asset Inventory", + "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", + "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", + "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html", + "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", + "release_level": "stable", + "transport": "grpc", + "requires_billing": true, + "language": "java", + "repo": "googleapis/java-asset", + "repo_short": "java-asset", + "distribution_name": "com.google.cloud:google-cloud-asset", + "library_type": "GAPIC_AUTO", + "api_id": "cloudasset.googleapis.com" +} \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/.repo-metadata.json b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/.repo-metadata.json new file mode 100644 index 0000000000..1c680ddfea --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/.repo-metadata.json @@ -0,0 +1,16 @@ +{ + "api_shortname": "admanager", + "name_pretty": "Google Ad Manager API", + "product_documentation": "https://developers.google.com/ad-manager/api/beta", + "api_description": "The Ad Manager API enables an app to integrate with Google Ad Manager. You can read Ad Manager data and run reports using the API.", + "client_documentation": "https://cloud.google.com/java/docs/reference/ad-manager/latest/overview", + "release_level": "preview", + "transport": "http", + "language": "java", + "repo": "googleapis/google-cloud-java", + "repo_short": "java-admanager", + "distribution_name": "com.google.api-ads:ad-manager", + "api_id": "admanager.googleapis.com", + "library_type": "GAPIC_AUTO", + "requires_billing": true +} \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/ad-manager-bom/pom-golden.xml b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/ad-manager-bom/pom-golden.xml new file mode 100644 index 0000000000..2adae69c60 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/ad-manager-bom/pom-golden.xml @@ -0,0 +1,38 @@ + + + 4.0.0 + com.google.api-ads + ad-manager-bom + 0.2.0-SNAPSHOT + pom + + com.google.cloud + google-cloud-pom-parent + 1.37.0-SNAPSHOT + ../../google-cloud-pom-parent/pom.xml + + + Google Google Ad Manager API BOM + + BOM for Google Ad Manager API + + + + true + + + + + + com.google.api-ads + ad-manager + 0.2.0-SNAPSHOT + + + com.google.api-ads.api.grpc + proto-ad-manager-v1 + 0.2.0-SNAPSHOT + + + + diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml new file mode 100644 index 0000000000..4c8a8a8343 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml @@ -0,0 +1,110 @@ + + + 4.0.0 + com.google.api-ads + ad-manager + 0.2.0-SNAPSHOT + jar + Google Google Ad Manager API + Google Ad Manager API The Ad Manager API enables an app to integrate with Google Ad Manager. You can read Ad Manager data and run reports using the API. + + com.google.api-ads + ad-manager-parent + 0.2.0-SNAPSHOT + + + ad-manager + + + + io.grpc + grpc-api + + + io.grpc + grpc-stub + + + io.grpc + grpc-protobuf + + + com.google.api + api-common + + + com.google.protobuf + protobuf-java + + + com.google.api.grpc + proto-google-common-protos + + + + com.google.api-ads.api.grpc + proto-ad-manager-v1 + + + com.google.guava + guava + + + com.google.api + gax + + + com.google.api + gax-grpc + + + com.google.api + gax-httpjson + + + com.google.api.grpc + grpc-google-common-protos + test + + + com.google.api.grpc + proto-google-iam-v1 + + + com.google.api.grpc + grpc-google-iam-v1 + test + + + org.threeten + threetenbp + + + + + junit + junit + test + + + + + com.google.api + gax + testlib + test + + + com.google.api + gax-grpc + testlib + test + + + com.google.api + gax-httpjson + testlib + test + + + diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/pom-golden.xml b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/pom-golden.xml new file mode 100644 index 0000000000..669d294ae7 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/pom-golden.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + com.google.api-ads + ad-manager-parent + pom + 0.2.0-SNAPSHOT + Google Google Ad Manager API Parent + + Java idiomatic client for Google Cloud Platform services. + + + + com.google.cloud + google-cloud-jar-parent + 1.37.0-SNAPSHOT + ../google-cloud-jar-parent/pom.xml + + + + UTF-8 + UTF-8 + github + ad-manager-parent + + + + + + com.google.api-ads + ad-manager + 0.2.0-SNAPSHOT + + + com.google.api-ads.api.grpc + proto-ad-manager-v1 + 0.2.0-SNAPSHOT + + + + + + + ad-manager + proto-ad-manager-v1 + ad-manager-bom + + + diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/proto-ad-manager-v1/pom-golden.xml b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/proto-ad-manager-v1/pom-golden.xml new file mode 100644 index 0000000000..eef103e533 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/proto-ad-manager-v1/pom-golden.xml @@ -0,0 +1,37 @@ + + 4.0.0 + com.google.api-ads.api.grpc + proto-ad-manager-v1 + 0.2.0-SNAPSHOT + proto-ad-manager-v1 + Proto library for ad-manager + + com.google.api-ads + ad-manager-parent + 0.2.0-SNAPSHOT + + + + com.google.protobuf + protobuf-java + + + com.google.api.grpc + proto-google-common-protos + + + com.google.api.grpc + proto-google-iam-v1 + + + com.google.api + api-common + + + com.google.guava + guava + + + diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt new file mode 100644 index 0000000000..3f073728f3 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/java-admanager/versions.txt @@ -0,0 +1,6 @@ +# Format: +# module:released-version:current-version + +google-cloud-java:1.36.0:1.37.0-SNAPSHOT +ad-manager:0.1.0:0.2.0-SNAPSHOT +proto-ad-manager-v1:0.1.0:0.2.0-SNAPSHOT diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/FooGrpcGolden.java b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/FooGrpcGolden.java new file mode 100644 index 0000000000..64f7bb90e2 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/FooGrpcGolden.java @@ -0,0 +1,22 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package foo; + +// This class is intentionally missing a license header + +class FooGrpc { + +} diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/FooProtoGolden.java b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/FooProtoGolden.java new file mode 100644 index 0000000000..75bfe578be --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/FooProtoGolden.java @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: foo/foo_proto.proto + +package foo; + +// This class is intentionally missing a license header + +class FooProto { + +} \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/README-golden.md b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/README-golden.md new file mode 100644 index 0000000000..bb4eb039f5 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/README-golden.md @@ -0,0 +1,202 @@ +# Google Cloud Asset Inventory Client for Java + +Java idiomatic client for [Cloud Asset Inventory][product-docs]. + +[![Maven][maven-version-image]][maven-version-link] +![Stability][stability-image] + +- [Product Documentation][product-docs] +- [Client Library Documentation][javadocs] + + +:bus: In October 2022, this library has moved to +[google-cloud-java/java-asset]( +https://github.com/googleapis/google-cloud-java/tree/main/java-asset). +This repository will be archived in the future. +Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). +The Maven artifact coordinates (`com.google.cloud:google-cloud-asset`) remain the same. + +## Quickstart + + +If you are using Maven, add this to your pom.xml file: + + +```xml + + com.google.cloud + google-cloud-asset + 1.2.3 + +``` + +If you are using Gradle without BOM, add this to your dependencies: + +```Groovy +implementation 'com.google.cloud:google-cloud-asset:1.2.3' +``` + +If you are using SBT, add this to your dependencies: + +```Scala +libraryDependencies += "com.google.cloud" % "google-cloud-asset" % "1.2.3" +``` + +## Authentication + +See the [Authentication][authentication] section in the base directory's README. + +## Authorization + +The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired Cloud Asset Inventory APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the Cloud Asset Inventory API calls. + +## Getting Started + +### Prerequisites + +You will need a [Google Cloud Platform Console][developer-console] project with the Cloud Asset Inventory [API enabled][enable-api]. +You will need to [enable billing][enable-billing] to use Google Cloud Asset Inventory. +[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by +[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: +`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +### Installation and setup + +You'll need to obtain the `google-cloud-asset` library. See the [Quickstart](#quickstart) section +to add `google-cloud-asset` as a dependency in your code. + +## About Cloud Asset Inventory + + +[Cloud Asset Inventory][product-docs] provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe. + +See the [Cloud Asset Inventory client library docs][javadocs] to learn how to +use this Cloud Asset Inventory Client Library. + + + + + + +## Troubleshooting + +To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. + +## Transport + +Cloud Asset Inventory uses gRPC for the transport layer. + +## Supported Java Versions + +Java 8 or above is required for using this client. + +Google's Java client libraries, +[Google Cloud Client Libraries][cloudlibs] +and +[Google Cloud API Libraries][apilibs], +follow the +[Oracle Java SE support roadmap][oracle] +(see the Oracle Java SE Product Releases section). + +### For new development + +In general, new feature development occurs with support for the lowest Java +LTS version covered by Oracle's Premier Support (which typically lasts 5 years +from initial General Availability). If the minimum required JVM for a given +library is changed, it is accompanied by a [semver][semver] major release. + +Java 11 and (in September 2021) Java 17 are the best choices for new +development. + +### Keeping production systems current + +Google tests its client libraries with all current LTS versions covered by +Oracle's Extended Support (which typically lasts 8 years from initial +General Availability). + +#### Legacy support + +Google's client libraries support legacy versions of Java runtimes with long +term stable libraries that don't receive feature updates on a best efforts basis +as it may not be possible to backport all patches. + +Google provides updates on a best efforts basis to apps that continue to use +Java 7, though apps might need to upgrade to current versions of the library +that supports their JVM. + +#### Where to find specific information + +The latest versions and the supported Java versions are identified on +the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` +and on [google-cloud-java][g-c-j]. + +## Versioning + + +This library follows [Semantic Versioning](http://semver.org/). + + + +## Contributing + + +Contributions to this library are always welcome and highly encouraged. + +See [CONTRIBUTING][contributing] for more information how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in +this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more +information. + + +## License + +Apache 2.0 - See [LICENSE][license] for more information. + +## CI Status + +Java Version | Status +------------ | ------ +Java 8 | [![Kokoro CI][kokoro-badge-image-2]][kokoro-badge-link-2] +Java 8 OSX | [![Kokoro CI][kokoro-badge-image-3]][kokoro-badge-link-3] +Java 8 Windows | [![Kokoro CI][kokoro-badge-image-4]][kokoro-badge-link-4] +Java 11 | [![Kokoro CI][kokoro-badge-image-5]][kokoro-badge-link-5] + +Java is a registered trademark of Oracle and/or its affiliates. + +[product-docs]: https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview +[javadocs]: https://googleapis.dev/java/google-cloud-asset/latest/index.html +[kokoro-badge-image-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java7.svg +[kokoro-badge-link-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java7.html +[kokoro-badge-image-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8.svg +[kokoro-badge-link-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8.html +[kokoro-badge-image-3]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8-osx.svg +[kokoro-badge-link-3]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8-osx.html +[kokoro-badge-image-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8-win.svg +[kokoro-badge-link-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java8-win.html +[kokoro-badge-image-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java11.svg +[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-asset/java11.html +[stability-image]: https://img.shields.io/badge/stability-stable-green +[maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-asset.svg +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-asset/1.2.3 +[authentication]: https://github.com/googleapis/google-cloud-java#authentication +[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes +[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles +[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy +[developer-console]: https://console.developers.google.com/ +[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[cloud-cli]: https://cloud.google.com/cli +[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md +[contributing]: https://github.com/googleapis/java-asset/blob/main/CONTRIBUTING.md +[code-of-conduct]: https://github.com/googleapis/java-asset/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[license]: https://github.com/googleapis/java-asset/blob/main/LICENSE +[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing +[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid=cloudasset.googleapis.com +[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png + +[semver]: https://semver.org/ +[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained +[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries +[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html +[g-c-j]: http://github.com/googleapis/google-cloud-java diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleClass.java b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleClass.java new file mode 100644 index 0000000000..849548b9da --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleClass.java @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp; + +class ExampleClass { + public static void foo() { + System.out.println("bar"); + } + + public static class InnerClass { + public void asdf() { + System.out.println("qwer"); + } + } +} diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleClassGolden.java b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleClassGolden.java new file mode 100644 index 0000000000..dbe1fe9d18 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleClassGolden.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp; + +class ExampleClass { + + public static class InnerClass { + } +} diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleCopyMethodGolden.java b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleCopyMethodGolden.java new file mode 100644 index 0000000000..02ce855bc9 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleCopyMethodGolden.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dlp; + +class ExampleClass { + public static void foo() { + System.out.println("bar"); + } + + public static void foobar() { + System.out.println("bar"); + } + + public static class InnerClass { + public void asdf() { + System.out.println("qwer"); + } + + public void xyz() { + System.out.println("qwer"); + } + } +} diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleDeprecateClass.java b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleDeprecateClass.java new file mode 100644 index 0000000000..d2ae795106 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleDeprecateClass.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +class ExampleClass { + public void cat(String bar) { + for (int i = 0; i < 3; i++) { + System.out.println("this is a test " + bar); + } + } + + @Beta + @Generated() + public void foo(String bar) { + for (int i = 0; i < 3; i++) { + System.out.println("this is a test " + bar); + } + } + + /** + * This is an existing comment. + */ + public void bar(String bar) { + for (int i = 0; i < 3; i++) { + System.out.println("this is a test " + bar); + } + } +} diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleDeprecateMethodGolden.java b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleDeprecateMethodGolden.java new file mode 100644 index 0000000000..a5107d9ce6 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/SampleDeprecateMethodGolden.java @@ -0,0 +1,52 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +class ExampleClass { + /** + * @deprecated sample has the same functionality as foobar. + */ + @Deprecated + public void cat(String bar) { + for (int i = 0; i < 3; i++) { + System.out.println("this is a test " + bar); + } + } + + /** + * @deprecated This method will be removed in the next major version. + * Use {@link #sample()} instead + */ + @Beta + @Generated() + @Deprecated + public void foo(String bar) { + for (int i = 0; i < 3; i++) { + System.out.println("this is a test " + bar); + } + } + + /** + * This is an existing comment. + * @deprecated This method will be removed in the next major version. + * Use {@link #sample()} instead + */ + @Deprecated + public void bar(String bar) { + for (int i = 0; i < 3; i++) { + System.out.println("this is a test " + bar); + } + } +} diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/src/foo/FooGrpc.java b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/src/foo/FooGrpc.java new file mode 100644 index 0000000000..be6abc0d64 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/src/foo/FooGrpc.java @@ -0,0 +1,7 @@ +package foo; + +// This class is intentionally missing a license header + +class FooGrpc { + +} diff --git a/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/src/foo/FooProto.java b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/src/foo/FooProto.java new file mode 100644 index 0000000000..c3aee1cffe --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test-owlbot/testdata/src/foo/FooProto.java @@ -0,0 +1,10 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: foo/foo_proto.proto + +package foo; + +// This class is intentionally missing a license header + +class FooProto { + +} \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml b/hermetic_build/library_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml new file mode 100644 index 0000000000..304ee9b892 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/gapic-libraries-bom/pom-golden.xml @@ -0,0 +1,45 @@ + + + 4.0.0 + com.google.cloud + gapic-libraries-bom + pom + 1.29.0-SNAPSHOT + Google Cloud Java BOM + + BOM for the libraries in google-cloud-java repository. Users should not + depend on this artifact explicitly because this BOM is an implementation + detail of the Libraries BOM. + + + + google-cloud-pom-parent + com.google.cloud + 1.29.0-SNAPSHOT + ../google-cloud-pom-parent/pom.xml + + + + + + com.google.cloud + google-cloud-dns + 2.33.0-SNAPSHOT + + + com.google.cloud + google-cloud-service-control-bom + 1.35.0-SNAPSHOT + pom + import + + + com.google.cloud + google-cloud-tasks-bom + 2.35.0-SNAPSHOT + pom + import + + + + \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-dns/pom.xml b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-dns/pom.xml new file mode 100644 index 0000000000..28bdaad76b --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-dns/pom.xml @@ -0,0 +1,9 @@ + + + 4.0.0 + com.google.cloud + google-cloud-dns + jar + 2.33.0-SNAPSHOT + Google Cloud DNS Parent + diff --git a/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml new file mode 100644 index 0000000000..483838475d --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-service-control/google-cloud-service-control-bom/pom.xml @@ -0,0 +1,8 @@ + + + 4.0.0 + com.google.cloud + google-cloud-service-control-bom + 1.35.0-SNAPSHOT + pom + diff --git a/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml new file mode 100644 index 0000000000..3138a26ce7 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/java-tasks/google-cloud-tasks-bom/pom.xml @@ -0,0 +1,8 @@ + + + 4.0.0 + com.google.cloud + google-cloud-tasks-bom + 2.35.0-SNAPSHOT + pom + diff --git a/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/pom-golden.xml b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/pom-golden.xml new file mode 100644 index 0000000000..d7c0a1f6a1 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/pom-golden.xml @@ -0,0 +1,88 @@ + + + 4.0.0 + google-cloud-java + com.google.cloud + 0.201.0 + pom + + + true + + + + gapic-libraries-bom + google-cloud-jar-parent + google-cloud-pom-parent + java-dns + java-service-control + java-tasks + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.3 + + true + + + + + + + + release-staging-repository + + + + !gpg.executable + + + + + sonatype-nexus-snapshots + https://google.oss.sonatype.org/content/repositories/snapshots + + + sonatype-nexus-staging + https://google.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.7.0 + true + + sonatype-nexus-staging + https://google.oss.sonatype.org/ + false + + + + + + + release-non-google-oss-sonatype + + + + org.sonatype.plugins + nexus-staging-maven-plugin + + ossrh + https://oss.sonatype.org/ + + + + + + + \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/versions.txt b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/versions.txt new file mode 100644 index 0000000000..6a537f4a39 --- /dev/null +++ b/hermetic_build/library_generation/tests/resources/test_monorepo_postprocessing/versions.txt @@ -0,0 +1,4 @@ +# Format: +# module:released-version:current-version + +google-cloud-java:1.28.0:1.29.0-SNAPSHOT diff --git a/hermetic_build/library_generation/tests/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt b/hermetic_build/library_generation/tests/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/main/java/example_main.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt b/hermetic_build/library_generation/tests/resources/test_mv_src/gapic/destination/java_gapic_srcjar/src/test/java/example_test.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt b/hermetic_build/library_generation/tests/resources/test_mv_src/proto/destination/java_gapic_srcjar/proto/src/main/java/example_proto_main.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt b/hermetic_build/library_generation/tests/resources/test_mv_src/samples/destination_com/java_gapic_srcjar/samples/snippets/generated/src/main/java/com/example_com_sample.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt b/hermetic_build/library_generation/tests/resources/test_mv_src/samples/destination_io/java_gapic_srcjar/samples/snippets/generated/src/main/java/io/example_io_sample.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/test_utilities.sh b/hermetic_build/library_generation/tests/test_utilities.sh new file mode 100755 index 0000000000..bbcdee1ebc --- /dev/null +++ b/hermetic_build/library_generation/tests/test_utilities.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +set -xeo pipefail +test_utilities_script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# Utility functions commonly used in test cases. + +# Variables used to generate final result +total_num=0 +succeed_num=0 +failed_num=0 +failed_tests="" + +############# Helper functions, they shouldn't be called outside this file ############# +__test_executed() { + total_num=$((1 + total_num)) +} + +__test_succeed() { + succeed_num=$((1 + succeed_num)) +} + +__test_failed() { + failed_test=$1 + failed_num=$((1 + failed_num)) + failed_tests="${failed_tests} ${failed_test}" +} + + +############# Functions used in test execution. They can only be called once per test ############# + +assertEquals() { + local expected=$1 + local actual=$2 + if [[ "${expected}" == "${actual}" ]]; then + __test_succeed + return + fi + + echo "Error: expected ${expected}, got ${actual} instead." + __test_failed "${ut}" +} + +assertFileOrDirectoryExists() { + local expected_file=$1 + if [ -d "${expected_file}" ]|| [ -f "${expected_file}" ]; then + __test_succeed + return + fi + + echo "Error: ${expected_file} does not exist." + __test_failed "${ut}" +} + +# Clean up generated files, tooling when testing `generate_library.sh`. +cleanup() { + local library_directory=$1 + rm -rf ../"${library_directory}" \ + google/protobuf \ + protobuf-* \ + gapic-generator-java-*.jar \ + gapic-generator-java-pom-parent-*.pom \ + protoc-gen-grpc-*.exe +} + +execute_tests() { + local test_list=("$@") + for ut in "${test_list[@]}"; do + echo "========== Execute ${ut} ==========" + __test_executed + "${ut}" + done + + echo "Test result: ${total_num} tests executed, ${succeed_num} succeed, ${failed_num} failed." + if [[ "${total_num}" == "${succeed_num}" ]]; then + echo "All tests passed." + exit + fi + + echo "Test failed." + echo "Failed test(s): ${failed_tests}." + exit 1 +} + +############# Utility functions used in `generate_library_integration_tests.sh` ############# + +# Obtains a version from a bazel WORKSPACE file +# +# versions look like "_ggj_version="1.2.3" +# It will return 1.2.3 for such example + +# performs a deep structural comparison between the current pom in a git +# folder and the one at HEAD. +# This function is OS-dependent, so it sources the main utilities script to +# perform detection +compare_poms() { + target_dir=$1 + source "${test_utilities_script_dir}/../utilities.sh" + os_architecture=$(detect_os_architecture) + pushd "${target_dir}" &> /dev/null + find . -name 'pom.xml' -exec cp {} {}.new \; + find . -name 'pom.xml' -exec git checkout HEAD -- {} \; + # compare_poms.py exits with non-zero if diffs are found + set -e + result=0 + if [ "${os_architecture}" == "linux-x86_64" ]; then + find . -name 'pom.xml' -print0 | xargs -i -0 python3 "${test_utilities_script_dir}/compare_poms.py" {} {}.new false || result=$? + else + find . -name 'pom.xml' -print0 | xargs -I{} -0 python3 "${test_utilities_script_dir}/compare_poms.py" {} {}.new false || result=$? + fi + popd &> /dev/null # target_dir + echo ${result} +} + +# computes the `destination_path` variable by inspecting the contents of the +# googleapis-gen at $proto_path. +compute_destination_path() { + local proto_path=$1 + local output_folder=$2 + pushd "${output_folder}" &> /dev/null + local destination_path=$(find "googleapis-gen/${proto_path}" -maxdepth 1 -name 'google-*-java' \ + | rev \ + | cut -d'/' -f1 \ + | rev + ) + popd &> /dev/null # output_folder + echo "${destination_path}" +} + diff --git a/hermetic_build/library_generation/tests/test_utils.py b/hermetic_build/library_generation/tests/test_utils.py new file mode 100644 index 0000000000..890cd95362 --- /dev/null +++ b/hermetic_build/library_generation/tests/test_utils.py @@ -0,0 +1,40 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from difflib import unified_diff +from pathlib import Path + +from typing import List + + +class FileComparator(unittest.TestCase): + def compare_files(self, expect: str, actual: str): + with open(expect, "r") as f: + expected_lines = f.readlines() + with open(actual, "r") as f: + actual_lines = f.readlines() + + diff = list(unified_diff(expected_lines, actual_lines)) + self.assertEqual( + first=[], second=diff, msg="Unexpected file contents:\n" + "".join(diff) + ) + + +def cleanup(files: List[str]): + for file in files: + path = Path(file).resolve() + if path.is_file(): + path.unlink() + elif path.is_dir(): + path.rmdir() diff --git a/hermetic_build/library_generation/tests/utilities_unit_tests.py b/hermetic_build/library_generation/tests/utilities_unit_tests.py new file mode 100644 index 0000000000..b8bb9c6149 --- /dev/null +++ b/hermetic_build/library_generation/tests/utilities_unit_tests.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Unit tests for python scripts +""" +import shutil +import unittest +import os +import io +import contextlib +from pathlib import Path +from library_generation.utils import utilities as util +from common.model.gapic_config import GapicConfig +from common.model.gapic_inputs import GapicInputs +from common.model.generation_config import GenerationConfig +from common.model.library_config import LibraryConfig +from library_generation.tests.test_utils import FileComparator +from library_generation.tests.test_utils import cleanup + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "resources") +file_comparator = FileComparator() +library_1 = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", + gapic_configs=list(), + library_name="bare-metal-solution", + rest_documentation="https://cloud.google.com/bare-metal/docs/reference/rest", + rpc_documentation="https://cloud.google.com/bare-metal/docs/reference/rpc", + recommended_package="com.google.example", + min_java_version=8, +) +library_2 = LibraryConfig( + api_shortname="secretmanager", + name_pretty="Secret Management", + product_documentation="https://cloud.google.com/solutions/secrets-management/", + api_description="allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", + gapic_configs=list(), +) +common_protos = LibraryConfig( + api_shortname="common-protos", + name_pretty="Common Protos", + product_documentation="", + api_description="example description", + gapic_configs=list(), +) +test_library_with_custom_transport = LibraryConfig( + api_shortname="secretmanager", + name_pretty="Secret Management", + product_documentation="https://cloud.google.com/solutions/secrets-management/", + api_description="allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", + gapic_configs=list(), + transport="rest", +) + + +class UtilitiesTest(unittest.TestCase): + """ + Unit tests for utilities.py + """ + + CONFIGURATION_YAML_PATH = os.path.join( + script_dir, + "resources", + "integration", + "google-cloud-java", + "generation_config.yaml", + ) + + def test_create_argument_valid_container_succeeds(self): + container_value = "google/test/v1" + container = GapicConfig(container_value) + argument_key = "proto_path" + result = util.create_argument(argument_key, container) + self.assertEqual([f"--{argument_key}", container_value], result) + + def test_create_argument_empty_container_returns_empty_list(self): + container = dict() + argument_key = "proto_path" + result = util.create_argument(argument_key, container) + self.assertEqual([], result) + + def test_create_argument_none_container_fails(self): + container = None + argument_key = "proto_path" + result = util.create_argument(argument_key, container) + self.assertEqual([], result) + + def test_sh_util_existent_function_succeeds(self): + result = util.sh_util("extract_folder_name path/to/folder_name") + self.assertEqual("folder_name", result) + + def test_sh_util_nonexistent_function_fails(self): + with self.assertRaises(RuntimeError): + result = util.sh_util("nonexistent_function") + + def test_mv_src_files_gapic_main_succeeds(self): + previous_dir = os.getcwd() + os.chdir(f"{resources_dir}/test_mv_src/gapic") + os.environ["folder_name"] = "example" + util.sh_util("mv_src_files gapic main destination") + self.assertTrue( + os.path.isfile("destination/gapic-example/src/main/java/example_main.txt") + ) + shutil.rmtree("destination/gapic-example") + os.chdir(previous_dir) + + def test_mv_src_files_gapic_test_succeeds(self): + previous_dir = os.getcwd() + os.chdir(f"{resources_dir}/test_mv_src/gapic") + os.environ["folder_name"] = "example" + util.sh_util("mv_src_files gapic test destination") + self.assertTrue( + os.path.isfile("destination/gapic-example/src/test/java/example_test.txt") + ) + shutil.rmtree("destination/gapic-example") + os.chdir(previous_dir) + + def test_mv_src_files_proto_main_succeeds(self): + previous_dir = os.getcwd() + os.chdir(f"{resources_dir}/test_mv_src/proto") + os.environ["folder_name"] = "example" + util.sh_util("mv_src_files proto main destination") + self.assertTrue( + os.path.isfile( + "destination/proto-example/src/main/java/example_proto_main.txt" + ) + ) + shutil.rmtree("destination/proto-example") + os.chdir(previous_dir) + + def test_mv_src_files_sample_suffix_io_succeeds(self): + previous_dir = os.getcwd() + os.chdir(f"{resources_dir}/test_mv_src/samples") + util.sh_util("mv_src_files samples main destination_io") + self.assertTrue( + os.path.isfile( + "destination_io/samples/snippets/generated/io/example_io_sample.txt" + ) + ) + shutil.rmtree("destination_io/samples") + os.chdir(previous_dir) + + def test_mv_src_files_sample_suffix_com_succeeds(self): + previous_dir = os.getcwd() + os.chdir(f"{resources_dir}/test_mv_src/samples") + util.sh_util("mv_src_files samples main destination_com") + self.assertTrue( + os.path.isfile( + "destination_com/samples/snippets/generated/com/example_com_sample.txt" + ) + ) + shutil.rmtree("destination_com/samples") + os.chdir(previous_dir) + + def test_eprint_valid_input_succeeds(self): + test_input = "This is some test input" + # create a stdio capture object + stderr_capture = io.StringIO() + # run eprint() with the capture object + with contextlib.redirect_stderr(stderr_capture): + util.eprint(test_input) + result = stderr_capture.getvalue() + # print() appends a `\n` each time it's called + self.assertEqual(test_input + "\n", result) + + def test_generate_postprocessing_prerequisite_files_non_monorepo_success(self): + library_path = self.__setup_postprocessing_prerequisite_files( + combination=1, library_type="GAPIC_COMBO" + ) + + file_comparator.compare_files( + f"{library_path}/.repo-metadata.json", + f"{library_path}/.repo-metadata-non-monorepo-golden.json", + ) + # since this is a single library, we treat this as HW repository, + # meaning that the owlbot yaml will be inside a .github folder + file_comparator.compare_files( + f"{library_path}/.github/.OwlBot-hermetic.yaml", + f"{library_path}/.OwlBot-hermetic-golden.yaml", + ) + file_comparator.compare_files( + f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" + ) + self.__remove_postprocessing_prerequisite_files( + path=library_path, is_monorepo=False + ) + + def test_generate_postprocessing_prerequisite_files_monorepo_success(self): + library_path = self.__setup_postprocessing_prerequisite_files(combination=2) + + file_comparator.compare_files( + f"{library_path}/.repo-metadata.json", + f"{library_path}/.repo-metadata-monorepo-golden.json", + ) + file_comparator.compare_files( + f"{library_path}/.OwlBot-hermetic.yaml", + f"{library_path}/.OwlBot-hermetic-golden.yaml", + ) + file_comparator.compare_files( + f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" + ) + self.__remove_postprocessing_prerequisite_files(path=library_path) + + def test_generate_postprocessing_prerequisite_files_proto_only_repo_success(self): + library_path = self.__setup_postprocessing_prerequisite_files( + combination=3, library_type="OTHER" + ) + + file_comparator.compare_files( + f"{library_path}/.repo-metadata.json", + f"{library_path}/.repo-metadata-proto-only-golden.json", + ) + file_comparator.compare_files( + f"{library_path}/.OwlBot-hermetic.yaml", + f"{library_path}/.OwlBot-hermetic-golden.yaml", + ) + file_comparator.compare_files( + f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" + ) + self.__remove_postprocessing_prerequisite_files(path=library_path) + + def test_generate_postprocessing_prerequisite_files__custom_transport_set_in_config__success( + self, + ): + """ + This test generates files for `test_library_with_custom_transport`, which + has an explicit value for transport declared (http). This is expected to + override the value obtained in BUILD.bazel via gapic_inputs.parse(). For + testing purposes, we test with a default GapicInputs object, whose transport + is set to "grpc". + """ + library_path = self.__setup_postprocessing_prerequisite_files( + combination=2, library=test_library_with_custom_transport + ) + + file_comparator.compare_files( + f"{library_path}/.repo-metadata.json", + f"{library_path}/.repo-metadata-custom-transport-golden.json", + ) + self.__remove_postprocessing_prerequisite_files(path=library_path) + + def test_create__library_invalid_transport__fails( + self, + ): + + with self.assertRaises(ValueError): + test_library_with_invalid_transport = LibraryConfig( + api_shortname="secretmanager", + name_pretty="Secret Management", + product_documentation="https://cloud.google.com/solutions/secrets-management/", + api_description="allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", + gapic_configs=list(), + transport="http", + ) + + def test_prepare_repo_monorepo_success(self): + gen_config = self.__get_a_gen_config(2) + repo_config = util.prepare_repo( + gen_config=gen_config, + library_config=gen_config.libraries, + repo_path=f"{resources_dir}/misc", + ) + self.assertEqual("output", Path(repo_config.output_folder).name) + library_path = sorted([Path(key).name for key in repo_config.libraries]) + self.assertEqual( + ["java-bare-metal-solution", "java-secretmanager"], library_path + ) + + def test_prepare_repo_monorepo_failed(self): + gen_config = self.__get_a_gen_config(2) + self.assertRaises( + FileNotFoundError, + util.prepare_repo, + gen_config, + gen_config.libraries, + f"{resources_dir}/non-exist", + ) + + def test_prepare_repo_split_repo_success(self): + gen_config = self.__get_a_gen_config(1) + repo_config = util.prepare_repo( + gen_config=gen_config, + library_config=gen_config.libraries, + repo_path=f"{resources_dir}/misc", + ) + self.assertEqual("output", Path(repo_config.output_folder).name) + library_path = sorted([Path(key).name for key in repo_config.libraries]) + self.assertEqual(["misc"], library_path) + shutil.rmtree(repo_config.output_folder) + + def test_get_java_generator_location_success(self): + location = util.sh_util("get_gapic_generator_location") + self.assertRegex(location, r"/.library_generation/gapic-generator-java.jar$") + + def test_get_java_formatter_location_success(self): + location = util.sh_util("get_java_formatter_location") + self.assertRegex(location, r"/.library_generation/google-java-format.jar$") + + def __setup_postprocessing_prerequisite_files( + self, + combination: int, + library_type: str = "GAPIC_AUTO", + library: LibraryConfig = library_1, + ) -> str: + library_path = f"{resources_dir}/goldens" + files = [ + f"{library_path}/.repo-metadata.json", + f"{library_path}/.OwlBot-hermetic.yaml", + f"{library_path}/owlbot.py", + ] + cleanup(files) + library.library_type = library_type + config = self.__get_a_gen_config(combination, library_type=library_type) + proto_path = "google/cloud/baremetalsolution/v2" + gapic_inputs = GapicInputs() # defaults to transport=grpc + transport = library.get_transport(gapic_inputs) + util.generate_postprocessing_prerequisite_files( + config=config, + library=library, + proto_path=proto_path, + transport=transport, + library_path=library_path, + ) + return library_path + + @staticmethod + def __get_a_gen_config( + combination: int, library_type: str = "GAPIC_AUTO" + ) -> GenerationConfig: + """ + Returns an object of GenerationConfig with one to three of + LibraryConfig objects. Other attributes are set to empty str. + + :param combination: combination of LibraryConfig objects associated with + the GenerationConfig. Only support 1, 2 or 3. + :return: an object of GenerationConfig + """ + if combination == 1: + libraries = [library_1] + elif combination == 2: + libraries = [library_1, library_2] + else: + libraries = [library_1, common_protos] + + # update libraries with custom configuration (for now, only + # library_type) + for library in libraries: + library.library_type = library_type + if combination == 1: + # treat this as a HW library case to generate a real-life + # repo-metadata + library.extra_versioned_modules = "test-module" + else: + library.extra_versioned_modules = None + + return GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + libraries=libraries, + ) + + @staticmethod + def __remove_postprocessing_prerequisite_files( + path: str, is_monorepo: bool = True + ) -> None: + os.remove(f"{path}/.repo-metadata.json") + os.remove(f"{path}/owlbot.py") + if is_monorepo: + os.remove(f"{path}/.OwlBot-hermetic.yaml") + return + if os.path.isdir(f"{path}/.github"): + shutil.rmtree(f"{path}/.github", ignore_errors=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/hermetic_build/library_generation/tests/utils/__init__.py b/hermetic_build/library_generation/tests/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py b/hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py new file mode 100644 index 0000000000..f88f71d40e --- /dev/null +++ b/hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py @@ -0,0 +1,506 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from common.model.gapic_config import GapicConfig +from common.model.generation_config import GenerationConfig +from common.model.library_config import LibraryConfig +from library_generation.utils.generation_config_comparator import ChangeType +from library_generation.utils.generation_config_comparator import compare_config + + +class GenerationConfigComparatorTest(unittest.TestCase): + def setUp(self) -> None: + self.baseline_library = LibraryConfig( + api_shortname="existing_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[], + ) + self.current_library = LibraryConfig( + api_shortname="existing_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[], + ) + self.baseline_config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + grpc_version="", + protoc_version="", + libraries=[self.baseline_library], + ) + self.current_config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + grpc_version="", + protoc_version="", + libraries=[self.current_library], + ) + + def test_compare_config_not_change(self): + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue(len(result.change_to_libraries) == 0) + + def test_compare_config_googleapis_update(self): + self.baseline_config.googleapis_commitish = ( + "1a45bf7393b52407188c82e63101db7dc9c72026" + ) + self.current_config.googleapis_commitish = ( + "1e6517ef4f949191c9e471857cf5811c8abcab84" + ) + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertEqual({ChangeType.GOOGLEAPIS_COMMIT: []}, result.change_to_libraries) + + def test_compare_config_generator_update(self): + self.baseline_config.gapic_generator_version = "1.2.3" + self.current_config.gapic_generator_version = "1.2.4" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("gapic_generator_version", config_change.changed_param) + self.assertEqual("1.2.4", config_change.current_value) + + def test_compare_config_libraries_bom_update(self): + self.baseline_config.libraries_bom_version = "26.36.0" + self.current_config.libraries_bom_version = "26.37.0" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("libraries_bom_version", config_change.changed_param) + self.assertEqual("26.37.0", config_change.current_value) + + def test_compare_protobuf_update(self): + self.baseline_config.protoc_version = "3.25.2" + self.current_config.protoc_version = "3.27.0" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("protoc_version", config_change.changed_param) + self.assertEqual("3.27.0", config_change.current_value) + + def test_compare_config_grpc_update(self): + self.baseline_config.grpc_version = "1.60.0" + self.current_config.grpc_version = "1.61.0" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("grpc_version", config_change.changed_param) + self.assertEqual("1.61.0", config_change.current_value) + + def test_compare_config_template_excludes_update(self): + self.baseline_config.template_excludes = [".github/*", ".kokoro/*"] + self.current_config.template_excludes = [ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + ] + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] + self.assertEqual("template_excludes", config_change.changed_param) + self.assertEqual( + [ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + ], + config_change.current_value, + ) + + def test_compare_config_library_addition(self): + self.current_config.libraries.append( + LibraryConfig( + api_shortname="new_library", + api_description="", + name_pretty="", + product_documentation="", + gapic_configs=[], + ) + ) + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] + self.assertEqual("new_library", config_change.library_name) + + def test_compare_config_library_removal_does_not_have_repo_or_library_level_change( + self, + ): + self.current_config.libraries = [] + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 0 + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 0 + ) + + def test_compare_config_api_shortname_update_without_library_name(self): + self.current_config.libraries[0].api_shortname = "new_api_shortname" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] + self.assertEqual("new_api_shortname", config_change.library_name) + + def test_compare_config_api_shortname_update_with_library_name_raise_error(self): + self.baseline_config.libraries[0].library_name = "old_library_name" + self.current_config.libraries[0].library_name = "old_library_name" + self.current_config.libraries[0].api_shortname = "new_api_shortname" + self.assertRaisesRegex( + ValueError, + r"api_shortname.*library_name", + compare_config, + self.baseline_config, + self.current_config, + ) + + def test_compare_config_library_name_update(self): + self.current_config.libraries[0].library_name = "new_library_name" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARIES_ADDITION][0] + self.assertEqual("new_library_name", config_change.library_name) + + def test_compare_config_api_description_update(self): + self.current_config.libraries[0].api_description = "updated description" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("api_description", config_change.changed_param) + self.assertEqual("updated description", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_name_pretty_update(self): + self.current_config.libraries[0].name_pretty = "new name" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("name_pretty", config_change.changed_param) + self.assertEqual("new name", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_product_docs_update(self): + self.current_config.libraries[0].product_documentation = "new docs" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("product_documentation", config_change.changed_param) + self.assertEqual("new docs", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_library_type_update(self): + self.current_config.libraries[0].library_type = "GAPIC_COMBO" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("library_type", config_change.changed_param) + self.assertEqual("GAPIC_COMBO", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_release_level_update(self): + self.current_config.libraries[0].release_level = "STABLE" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("release_level", config_change.changed_param) + self.assertEqual("STABLE", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_api_id_update(self): + self.current_config.libraries[0].api_id = "new_id" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("api_id", config_change.changed_param) + self.assertEqual("new_id", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_api_reference_update(self): + self.current_config.libraries[0].api_reference = "new api_reference" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("api_reference", config_change.changed_param) + self.assertEqual("new api_reference", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_code_owner_team_update(self): + self.current_config.libraries[0].codeowner_team = "new team" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("codeowner_team", config_change.changed_param) + self.assertEqual("new team", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_excluded_deps_update(self): + self.current_config.libraries[0].excluded_dependencies = "group:artifact" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("excluded_dependencies", config_change.changed_param) + self.assertEqual("group:artifact", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_excluded_poms_update(self): + self.current_config.libraries[0].excluded_poms = "pom.xml" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("excluded_poms", config_change.changed_param) + self.assertEqual("pom.xml", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_client_docs_update(self): + self.current_config.libraries[0].client_documentation = "new client docs" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("client_documentation", config_change.changed_param) + self.assertEqual("new client docs", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_distribution_name_update(self): + self.current_config.libraries[0].distribution_name = "new_group:new_artifact" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("distribution_name", config_change.changed_param) + self.assertEqual("new_group:new_artifact", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_group_id_update(self): + self.current_config.libraries[0].group_id = "new_group" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("group_id", config_change.changed_param) + self.assertEqual("new_group", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_issue_tracker_update(self): + self.current_config.libraries[0].issue_tracker = "new issue tracker" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("issue_tracker", config_change.changed_param) + self.assertEqual("new issue tracker", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_rest_docs_update(self): + self.current_config.libraries[0].rest_documentation = "new rest docs" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("rest_documentation", config_change.changed_param) + self.assertEqual("new rest docs", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_rpc_docs_update(self): + self.current_config.libraries[0].rpc_documentation = "new rpc docs" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("rpc_documentation", config_change.changed_param) + self.assertEqual("new rpc docs", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_cloud_api_update(self): + self.current_config.libraries[0].cloud_api = False + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("cloud_api", config_change.changed_param) + self.assertEqual(False, config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_requires_billing_update(self): + self.current_config.libraries[0].requires_billing = False + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("requires_billing", config_change.changed_param) + self.assertEqual(False, config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_extra_versioned_mod_update(self): + self.current_config.libraries[0].extra_versioned_modules = "extra module" + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue( + len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 + ) + config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] + self.assertEqual("extra_versioned_modules", config_change.changed_param) + self.assertEqual("extra module", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) + + def test_compare_config_version_addition(self): + self.current_config.libraries[0].gapic_configs = [ + GapicConfig(proto_path="google/new/library/v1") + ] + result = compare_config( + baseline_config=self.baseline_config, + current_config=self.current_config, + ) + self.assertTrue(len(result.change_to_libraries[ChangeType.GAPIC_ADDITION]) == 1) + config_change = result.change_to_libraries[ChangeType.GAPIC_ADDITION][0] + self.assertEqual("", config_change.changed_param) + self.assertEqual("google/new/library/v1", config_change.current_value) + self.assertEqual("existing_library", config_change.library_name) diff --git a/hermetic_build/library_generation/tests/utils/monorepo_postprocessor_unit_tests.py b/hermetic_build/library_generation/tests/utils/monorepo_postprocessor_unit_tests.py new file mode 100644 index 0000000000..b95708c2ff --- /dev/null +++ b/hermetic_build/library_generation/tests/utils/monorepo_postprocessor_unit_tests.py @@ -0,0 +1,46 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest + +from library_generation.tests.test_utils import FileComparator +from library_generation.tests.test_utils import cleanup +from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") +file_comparator = FileComparator() + + +class MonorepoPostprocessorTest(unittest.TestCase): + def test_monorepo_postprocessing_valid_repository_success(self): + repository_path = f"{resources_dir}/test_monorepo_postprocessing" + versions_file = f"{repository_path}/versions.txt" + files = [ + f"{repository_path}/pom.xml", + f"{repository_path}/gapic-libraries-bom/pom.xml", + ] + cleanup(files) + monorepo_postprocessing( + repository_path=repository_path, versions_file=versions_file + ) + file_comparator.compare_files( + expect=f"{repository_path}/pom-golden.xml", + actual=f"{repository_path}/pom.xml", + ) + file_comparator.compare_files( + expect=f"{repository_path}/gapic-libraries-bom/pom-golden.xml", + actual=f"{repository_path}/gapic-libraries-bom/pom.xml", + ) + cleanup(files) diff --git a/hermetic_build/library_generation/tests/utils/pom_generator_unit_tests.py b/hermetic_build/library_generation/tests/utils/pom_generator_unit_tests.py new file mode 100644 index 0000000000..7a829e58aa --- /dev/null +++ b/hermetic_build/library_generation/tests/utils/pom_generator_unit_tests.py @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest + +from library_generation.utils.pom_generator import get_version_from + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") + + +class PomGeneratorTest(unittest.TestCase): + def test_get_version_from_returns_current(self): + versions_file = f"{resources_dir}/misc/versions.txt" + artifact = "gax-grpc" + self.assertEqual("2.33.1-SNAPSHOT", get_version_from(versions_file, artifact)) + + def test_get_version_from_returns_released(self): + versions_file = f"{resources_dir}/misc/versions.txt" + artifact = "gax-grpc" + self.assertEqual("2.34.0", get_version_from(versions_file, artifact, True)) diff --git a/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py b/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py new file mode 100644 index 0000000000..2e23c2b403 --- /dev/null +++ b/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +from pathlib import Path +from library_generation.utils.proto_path_utils import ( + find_versioned_proto_path, + remove_version_from, +) + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") +test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() + + +class ProtoPathsUtilsTest(unittest.TestCase): + def test_find_versioned_proto_path_nested_version_success(self): + proto_path = "google/cloud/aiplatform/v1/schema/predict/params/image_classification.proto" + expected = "google/cloud/aiplatform/v1" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_find_versioned_proto_path_success(self): + proto_path = "google/cloud/asset/v1p2beta1/assets.proto" + expected = "google/cloud/asset/v1p2beta1" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_find_versioned_proto_without_version_return_itself(self): + proto_path = "google/type/color.proto" + expected = "google/type/color.proto" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_remove_version_from_returns_non_versioned_path(self): + proto_path = "google/cloud/aiplatform/v1" + self.assertEqual("google/cloud/aiplatform", remove_version_from(proto_path)) + + def test_remove_version_from_returns_self(self): + proto_path = "google/cloud/aiplatform" + self.assertEqual("google/cloud/aiplatform", remove_version_from(proto_path)) diff --git a/hermetic_build/library_generation/utils/file_render.py b/hermetic_build/library_generation/utils/file_render.py new file mode 100644 index 0000000000..5c68445753 --- /dev/null +++ b/hermetic_build/library_generation/utils/file_render.py @@ -0,0 +1,24 @@ +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from jinja2 import Environment, FileSystemLoader + +script_dir = os.path.dirname(os.path.realpath(__file__)) +jinja_env = Environment(loader=FileSystemLoader(f"{script_dir}/../templates")) + + +def render(template_name: str, output_name: str, **kwargs): + template = jinja_env.get_template(template_name) + t = template.stream(kwargs) + directory = os.path.dirname(output_name) + if not os.path.isdir(directory): + os.makedirs(directory) + t.dump(str(output_name)) diff --git a/hermetic_build/library_generation/utils/generation_config_comparator.py b/hermetic_build/library_generation/utils/generation_config_comparator.py new file mode 100644 index 0000000000..d0851c7f31 --- /dev/null +++ b/hermetic_build/library_generation/utils/generation_config_comparator.py @@ -0,0 +1,284 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import defaultdict +from typing import Any +from typing import Dict +from typing import List +from library_generation.model.config_change import ChangeType +from library_generation.model.config_change import ConfigChange +from library_generation.model.config_change import LibraryChange +from library_generation.model.config_change import HashLibrary +from common.model.gapic_config import GapicConfig +from common.model.generation_config import GenerationConfig +from common.model.library_config import LibraryConfig + + +def compare_config( + baseline_config: GenerationConfig, current_config: GenerationConfig +) -> ConfigChange: + """ + Compare two GenerationConfig object and output a mapping from ConfigChange + to a list of ConfigChange objects. + All libraries in the current configuration will be affected if one of the + repository level parameters is changed. + + :param baseline_config: the baseline GenerationConfig object + :param current_config: the current GenerationConfig object + :return: a ConfigChange objects. + """ + diff = defaultdict(list[LibraryChange]) + # Exclude `libraries` because an empty library list (e.g., by library + # removal) will cause this parameter appears in the sorted param list, + # which leads to unequal list of parameters. + excluded_params = {"libraries"} + baseline_params = __convert_params_to_sorted_list( + obj=baseline_config, excluded_params=excluded_params + ) + current_params = __convert_params_to_sorted_list( + obj=current_config, excluded_params=excluded_params + ) + + for baseline_param, current_param in zip(baseline_params, current_params): + if baseline_param == current_param: + continue + if baseline_param[0] == "googleapis_commitish": + diff[ChangeType.GOOGLEAPIS_COMMIT] = [] + else: + config_change = LibraryChange( + changed_param=current_param[0], + current_value=current_param[1], + ) + diff[ChangeType.REPO_LEVEL_CHANGE].append(config_change) + + __compare_libraries( + diff=diff, + baseline_library_configs=baseline_config.libraries, + current_library_configs=current_config.libraries, + ) + return ConfigChange( + change_to_libraries=diff, + baseline_config=baseline_config, + current_config=current_config, + ) + + +def __compare_libraries( + diff: Dict[ChangeType, list[LibraryChange]], + baseline_library_configs: List[LibraryConfig], + current_library_configs: List[LibraryConfig], +) -> None: + """ + Compare two lists of LibraryConfig and put the difference into a + given Dict. + + :param diff: a mapping from ConfigChange to a list of ConfigChange objects. + :param baseline_library_configs: a list of LibraryConfig object. + :param current_library_configs: a list of LibraryConfig object. + """ + baseline_libraries = __convert_to_hashed_library_dict(baseline_library_configs) + current_libraries = __convert_to_hashed_library_dict(current_library_configs) + changed_libraries = [] + # 1st round comparison. + for library_name, hash_library in baseline_libraries.items(): + # 1. find any library removed from baseline_libraries. + # a library is removed from baseline_libraries if the library_name + # is not in current_libraries. + # please see the reason of comment out these lines of code in the + # comment of ChangeType.LIBRARIES_REMOVAL. + # if library_name not in current_libraries: + # config_change = ConfigChange( + # changed_param="", current_value="", library_name=library_name + # ) + # diff[ChangeType.LIBRARIES_REMOVAL].append(config_change) + + # 2. find any library that exists in both configs but at least one + # parameter is changed, which means the hash value is different. + if ( + library_name in current_libraries + and hash_library.hash_value != current_libraries[library_name].hash_value + ): + changed_libraries.append(library_name) + # 2nd round comparison. + for library_name in current_libraries: + # find any library added to current_libraries. + # a library is added to current_libraries if the library_name + # is not in baseline_libraries. + if library_name not in baseline_libraries: + config_change = LibraryChange( + changed_param="", current_value="", library_name=library_name + ) + diff[ChangeType.LIBRARIES_ADDITION].append(config_change) + # 3rd round comparison. + __compare_changed_libraries( + diff=diff, + baseline_libraries=baseline_libraries, + current_libraries=current_libraries, + changed_libraries=changed_libraries, + ) + + +def __convert_to_hashed_library_dict( + libraries: List[LibraryConfig], +) -> Dict[str, HashLibrary]: + """ + Convert a list of LibraryConfig objects to a Dict. + For each library object, the key is the library_name of the object, the + value is a HashLibrary object. + + :param libraries: a list of LibraryConfig object. + :return: a mapping from library_name to HashLibrary object. + """ + return { + library.get_library_name(): HashLibrary(hash(library), library) + for library in libraries + } + + +def __compare_changed_libraries( + diff: Dict[ChangeType, list[LibraryChange]], + baseline_libraries: Dict[str, HashLibrary], + current_libraries: Dict[str, HashLibrary], + changed_libraries: List[str], +) -> None: + """ + Compare each library with the same library_name to find what parameters are + changed. + + :param diff: a mapping from ConfigChange to a list of ConfigChange objects. + :param baseline_libraries: a mapping from library_name to HashLibrary + object. + :param current_libraries: a mapping from library_name to HashLibrary object. + :param changed_libraries: a list of library_name of changed libraries. + :raise ValueError: if api_shortname of a library is changed but library_name + remains the same. + """ + for library_name in changed_libraries: + baseline_library = baseline_libraries[library_name].library + current_library = current_libraries[library_name].library + baseline_params = __convert_params_to_sorted_list(baseline_library) + current_params = __convert_params_to_sorted_list(current_library) + for baseline_param, current_param in zip(baseline_params, current_params): + if baseline_param == current_param: + continue + if baseline_param[0] == "api_shortname": + raise ValueError( + f"{library_name}: api_shortname must not change when library_name remains the same." + ) + else: + config_change = LibraryChange( + changed_param=current_param[0], + current_value=current_param[1], + library_name=library_name, + ) + diff[ChangeType.LIBRARY_LEVEL_CHANGE].append(config_change) + + # compare gapic_configs + baseline_gapic_configs = baseline_library.gapic_configs + current_gapic_configs = current_library.gapic_configs + __compare_gapic_configs( + diff=diff, + library_name=library_name, + baseline_gapic_configs=baseline_gapic_configs, + current_gapic_configs=current_gapic_configs, + ) + + +def __compare_gapic_configs( + diff: Dict[ChangeType, list[LibraryChange]], + library_name: str, + baseline_gapic_configs: List[GapicConfig], + current_gapic_configs: List[GapicConfig], +) -> None: + baseline_proto_paths = {config.proto_path for config in baseline_gapic_configs} + current_proto_paths = {config.proto_path for config in current_gapic_configs} + # 1st round of comparison, find any versioned proto_path is removed + # from baseline gapic configs. + # please see the reason of comment out these lines of code in the + # comment of ChangeType.GAPIC_REMOVAL. + # for proto_path in baseline_proto_paths: + # if proto_path in current_proto_paths: + # continue + # config_change = ConfigChange( + # changed_param="", current_value=proto_path, library_name=library_name + # ) + # diff[ChangeType.GAPIC_REMOVAL].append(config_change) + + # 2nd round of comparison, find any versioned proto_path is added + # to current gapic configs. + for proto_path in current_proto_paths: + if proto_path in baseline_proto_paths: + continue + config_change = LibraryChange( + changed_param="", current_value=proto_path, library_name=library_name + ) + diff[ChangeType.GAPIC_ADDITION].append(config_change) + + +def __convert_params_to_sorted_list(obj: Any, excluded_params=None) -> List[tuple]: + """ + Convert the parameter and its value of a given object to a sorted list of + tuples. + + Only the following types of parameters will be considered: + + - str + - bool + - list[str] + - None + + Note that built-in params, e.g., __str__, and methods will be skipped. + + :param obj: an object. + :param excluded_params: excluded params. + :return: a sorted list of tuples. + """ + if excluded_params is None: + excluded_params = set() + param_and_values = [] + for param, value in vars(obj).items(): + if ( + param in excluded_params + or param.startswith("__") # skip built-in params + or callable(getattr(obj, param)) # skip methods + # skip if the type of param is not one of the following types + # 1. str + # 2. bool + # 3. list[str] + # 4. None + or not ( + isinstance(getattr(obj, param), str) + or isinstance(getattr(obj, param), bool) + or __is_list_of_str(obj=obj, param=param) + or getattr(obj, param) is None + ) + ): + continue + param_and_values.append((param, value)) + return sorted(param_and_values) + + +def __is_list_of_str(obj: Any, param: str) -> bool: + """ + Returns True if the type of param of a given object is a list of str; False + otherwise. + + This method is a workaround of https://bugs.python.org/issue28339. + """ + value = getattr(obj, param) + if not isinstance(value, list): + return False + for v in value: + if not isinstance(v, str): + return False + return True diff --git a/hermetic_build/library_generation/utils/monorepo_postprocessor.py b/hermetic_build/library_generation/utils/monorepo_postprocessor.py new file mode 100644 index 0000000000..cd522b83b5 --- /dev/null +++ b/hermetic_build/library_generation/utils/monorepo_postprocessor.py @@ -0,0 +1,29 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from library_generation.utils.pom_generator import generate_gapic_bom +from library_generation.utils.pom_generator import generate_root_pom + + +def monorepo_postprocessing( + repository_path: str, + versions_file: str, +) -> None: + """ + Perform repository level post-processing + :param repository_path: the path of the repository + :param versions_file: the versions_txt contains version of modules + :return: None + """ + generate_root_pom(repository_path=repository_path) + generate_gapic_bom(repository_path=repository_path, versions_file=versions_file) diff --git a/hermetic_build/library_generation/utils/pom_generator.py b/hermetic_build/library_generation/utils/pom_generator.py new file mode 100644 index 0000000000..8312d4c5a4 --- /dev/null +++ b/hermetic_build/library_generation/utils/pom_generator.py @@ -0,0 +1,157 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pathlib import Path + +from lxml import etree +from typing import List + +from library_generation.model.bom_config import BomConfig +from library_generation.utils.file_render import render + +project_tag = "{http://maven.apache.org/POM/4.0.0}" +group_id_tag = "groupId" +artifact_tag = "artifactId" +version_tag = "version" + + +def generate_root_pom(repository_path: str) -> None: + print("Regenerating root pom.xml") + modules = __search_for_java_modules(repository_path) + render( + template_name="root-pom.xml.j2", + output_name=f"{repository_path}/pom.xml", + modules=modules, + ) + + +def generate_gapic_bom(repository_path: str, versions_file: str) -> None: + print("Regenerating gapic-libraries-bom") + bom_configs = __search_for_bom_artifact(repository_path) + monorepo_version = get_version_from( + versions_file=versions_file, + artifact_id="google-cloud-java", + ) + render( + template_name="gapic-libraries-bom.xml.j2", + output_name=f"{repository_path}/gapic-libraries-bom/pom.xml", + monorepo_version=monorepo_version, + bom_configs=bom_configs, + ) + + +def get_version_from( + versions_file: str, artifact_id: str, is_released: bool = False +) -> str: + """ + Get version of a given artifact from versions.txt + :param versions_file: the path of version.txt + :param artifact_id: the artifact id + :param is_released: whether returns the released or current version + :return: the version of the artifact + """ + index = 1 if is_released else 2 + with open(versions_file, "r") as f: + for line in f.readlines(): + if artifact_id in line: + return line.split(":")[index].strip() + + +def __search_for_java_modules( + repository_path: str, +) -> List[str]: + repo = Path(repository_path).resolve() + modules = [] + for sub_dir in repo.iterdir(): + if sub_dir.is_dir() and sub_dir.name.startswith("java-"): + modules.append(sub_dir.name) + return sorted(modules) + + +def __search_for_bom_artifact( + repository_path: str, +) -> List[BomConfig]: + repo = Path(repository_path).resolve() + module_exclusions = ["gapic-libraries-bom"] + group_id_inclusions = [ + "com.google.cloud", + "com.google.analytics", + "com.google.area120", + ] + bom_configs = [] + for module in repo.iterdir(): + if module.is_file() or module.name in module_exclusions: + continue + for sub_module in module.iterdir(): + if sub_module.is_dir() and sub_module.name.endswith("-bom"): + root = etree.parse(f"{sub_module}/pom.xml").getroot() + group_id = root.find(f"{project_tag}{group_id_tag}").text + if group_id not in group_id_inclusions: + continue + artifact_id = root.find(f"{project_tag}{artifact_tag}").text + version = root.find(f"{project_tag}{version_tag}").text + index = artifact_id.rfind("-") + version_annotation = artifact_id[:index] + bom_configs.append( + BomConfig( + group_id=group_id, + artifact_id=artifact_id, + version=version, + version_annotation=version_annotation, + ) + ) + # handle edge case: java-grafeas + bom_configs += __handle_special_bom( + repository_path=repository_path, + module="java-grafeas", + group_id="io.grafeas", + artifact_id="grafeas", + ) + # handle edge case: java-dns + bom_configs += __handle_special_bom( + repository_path=repository_path, + module="java-dns", + group_id="com.google.cloud", + artifact_id="google-cloud-dns", + ) + # handle edge case: java-notification + bom_configs += __handle_special_bom( + repository_path=repository_path, + module="java-notification", + group_id="com.google.cloud", + artifact_id="google-cloud-notification", + ) + + return sorted(bom_configs) + + +def __handle_special_bom( + repository_path: str, + module: str, + group_id: str, + artifact_id: str, +) -> List[BomConfig]: + pom = f"{repository_path}/{module}/pom.xml" + if not Path(pom).exists(): + return [] + root = etree.parse(pom).getroot() + version = root.find(f"{project_tag}{version_tag}").text + return [ + BomConfig( + group_id=group_id, + artifact_id=artifact_id, + version=version, + version_annotation=artifact_id, + is_import=False, + ) + ] diff --git a/hermetic_build/library_generation/utils/proto_path_utils.py b/hermetic_build/library_generation/utils/proto_path_utils.py new file mode 100644 index 0000000000..d2ae25f602 --- /dev/null +++ b/hermetic_build/library_generation/utils/proto_path_utils.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + + +def remove_version_from(proto_path: str) -> str: + """ + Remove the version of a proto_path + :param proto_path: versioned proto_path + :return: the proto_path without version + """ + version_pattern = "^v[1-9]" + index = proto_path.rfind("/") + version = proto_path[index + 1 :] + if re.match(version_pattern, version): + return proto_path[:index] + return proto_path + + +def find_versioned_proto_path(proto_path: str) -> str: + """ + Returns a versioned proto_path from a given proto_path; or proto_path itself + if it doesn't contain a versioned proto_path. + :param proto_path: a proto file path + :return: the versioned proto_path + """ + version_regex = re.compile(r"^v[1-9].*") + directories = proto_path.split("/") + for directory in directories: + result = version_regex.search(directory) + if result: + version = result[0] + idx = proto_path.find(version) + return proto_path[:idx] + version + return proto_path diff --git a/hermetic_build/library_generation/utils/utilities.py b/hermetic_build/library_generation/utils/utilities.py new file mode 100755 index 0000000000..c75d3957ad --- /dev/null +++ b/hermetic_build/library_generation/utils/utilities.py @@ -0,0 +1,309 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import sys +import subprocess +import os +from pathlib import Path +from typing import Any + +from common.model.generation_config import GenerationConfig +from common.model.library_config import LibraryConfig +from typing import List +from library_generation.model.repo_config import RepoConfig +from library_generation.utils.file_render import render +from library_generation.utils.proto_path_utils import remove_version_from + +script_dir = os.path.dirname(os.path.realpath(__file__)) +SDK_PLATFORM_JAVA = "googleapis/sdk-platform-java" + + +def create_argument(arg_key: str, arg_container: object) -> List[str]: + """ + Generates a list of two elements [argument, value], or returns + an empty array if arg_val is None + """ + arg_val = getattr(arg_container, arg_key, None) + if arg_val is not None: + return [f"--{arg_key}", f"{arg_val}"] + return [] + + +def run_process_and_print_output( + arguments: List[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs +) -> Any: + """ + Runs a process with the given "arguments" list and prints its output. + If the process fails, then the whole program exits + :param arguments: list of strings or string containing the arguments + :param job_name: optional job name to be printed + :param exit_on_fail: True if the main program should exit when the + subprocess exits with non-zero. Used for testing. + :param kwargs: passed to the underlying subprocess.run() call + """ + # split "arguments" if passed as a single string + if isinstance(arguments, str): + arguments = arguments.split(" ") + if "stderr" not in kwargs: + kwargs["stderr"] = subprocess.STDOUT + proc_info = subprocess.run(arguments, stdout=subprocess.PIPE, **kwargs) + print(proc_info.stdout.decode(), end="", flush=True) + print( + f"{job_name} {'finished successfully' if proc_info.returncode == 0 else 'failed'}" + ) + if exit_on_fail and proc_info.returncode != 0: + sys.exit(1) + return proc_info + + +def run_process_and_get_output_string( + arguments: List[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs +) -> Any: + """ + Wrapper of run_process_and_print_output() that returns the merged + stdout and stderr in a single string without its trailing newline char. + """ + return run_process_and_print_output( + arguments, job_name, exit_on_fail, stderr=subprocess.PIPE, **kwargs + ).stdout.decode()[:-1] + + +def sh_util(statement: str, **kwargs) -> str: + """ + Calls a function defined in library_generation/utils/utilities.sh + """ + if "stdout" not in kwargs: + kwargs["stdout"] = subprocess.PIPE + if "stderr" not in kwargs: + kwargs["stderr"] = subprocess.PIPE + output = "" + with subprocess.Popen( + [ + "bash", + "-exc", + f"source {script_dir}/utilities.sh && {statement}", + ], + **kwargs, + ) as proc: + print("command stderr:") + for line in proc.stderr: + print(line.decode(), end="", flush=True) + print("command stdout:") + for line in proc.stdout: + print(line.decode(), end="", flush=True) + output += line.decode() + proc.wait() + if proc.returncode != 0: + raise RuntimeError( + f"function {statement} failed with exit code {proc.returncode}" + ) + # captured stdout may contain a newline at the end, we remove it + if len(output) > 0 and output[-1] == "\n": + output = output[:-1] + return output + + +def eprint(*args, **kwargs): + """ + prints to stderr + """ + print(*args, file=sys.stderr, **kwargs) + + +def prepare_repo( + gen_config: GenerationConfig, + library_config: List[LibraryConfig], + repo_path: str, + language: str = "java", +) -> RepoConfig: + """ + Gather information of the generated repository. + + :param gen_config: a GenerationConfig object representing a parsed + configuration yaml + :param library_config: a LibraryConfig object contained inside config, + passed here for convenience and to prevent all libraries to be processed + :param repo_path: the path to which the generated repository goes + :param language: programming language of the library + :return: a RepoConfig object contained repository information + :raise FileNotFoundError if there's no versions.txt in repo_path + :raise ValueError if two libraries have the same library_name + """ + output_folder = sh_util("get_output_folder") + print(f"output_folder: {output_folder}") + os.makedirs(output_folder, exist_ok=True) + libraries = {} + for library in library_config: + library_name = f"{language}-{library.get_library_name()}" + library_path = ( + f"{repo_path}/{library_name}" + if gen_config.is_monorepo() + else f"{repo_path}" + ) + # use absolute path because docker requires absolute path + # in volume name. + absolute_library_path = str(Path(library_path).resolve()) + libraries[absolute_library_path] = library + # remove existing .repo-metadata.json + json_name = ".repo-metadata.json" + if os.path.exists(f"{absolute_library_path}/{json_name}"): + os.remove(f"{absolute_library_path}/{json_name}") + versions_file = f"{repo_path}/versions.txt" + if not Path(versions_file).exists(): + raise FileNotFoundError(f"{versions_file} is not found.") + + return RepoConfig( + output_folder=output_folder, + libraries=libraries, + versions_file=str(Path(versions_file).resolve()), + ) + + +def generate_postprocessing_prerequisite_files( + config: GenerationConfig, + library: LibraryConfig, + proto_path: str, + transport: str, + library_path: str, + language: str = "java", +) -> None: + """ + Generates the postprocessing prerequisite files for a library. + + :param config: a GenerationConfig object representing a parsed configuration + yaml + :param library: the library configuration + :param proto_path: the path from the root of googleapis to the location of the service + protos. If the path contains a version, it will be removed + :param transport: transport supported by the library + :param library_path: the path to which the generated file goes + :param language: programming language of the library + :return: None + """ + library_name = library.get_library_name() + artifact_id = library.get_artifact_id() + if config.contains_common_protos(): + repo = SDK_PLATFORM_JAVA + elif config.is_monorepo(): + repo = "googleapis/google-cloud-java" + else: + repo = f"googleapis/{language}-{library_name}" + api_id = ( + library.api_id if library.api_id else f"{library.api_shortname}.googleapis.com" + ) + client_documentation = ( + library.client_documentation + if library.client_documentation + else f"https://cloud.google.com/{language}/docs/reference/{artifact_id}/latest/overview" + ) + + # The mapping is needed because transport in .repo-metadata.json + # is one of grpc, http and both, + if transport == "grpc": + converted_transport = "grpc" + elif transport == "rest": + converted_transport = "http" + else: + converted_transport = "both" + + repo_metadata = { + "api_shortname": library.api_shortname, + "name_pretty": library.name_pretty, + "product_documentation": library.product_documentation, + "api_description": library.api_description, + "client_documentation": client_documentation, + "release_level": library.release_level, + "transport": converted_transport, + "language": language, + "repo": repo, + "repo_short": f"{language}-{library_name}", + "distribution_name": library.get_maven_coordinate(), + "api_id": api_id, + "library_type": library.library_type, + "requires_billing": library.requires_billing, + } + + # this removal is for java-common-protos and java-iam in + # sdk-platform-java + if repo == SDK_PLATFORM_JAVA: + repo_metadata.pop("api_id") + + if library.api_reference: + repo_metadata["api_reference"] = library.api_reference + if library.codeowner_team: + repo_metadata["codeowner_team"] = library.codeowner_team + if library.excluded_dependencies: + repo_metadata["excluded_dependencies"] = library.excluded_dependencies + if library.excluded_poms: + repo_metadata["excluded_poms"] = library.excluded_poms + if library.issue_tracker: + repo_metadata["issue_tracker"] = library.issue_tracker + if library.rest_documentation: + repo_metadata["rest_documentation"] = library.rest_documentation + if library.rpc_documentation: + repo_metadata["rpc_documentation"] = library.rpc_documentation + if library.extra_versioned_modules: + repo_metadata["extra_versioned_modules"] = library.extra_versioned_modules + if library.recommended_package: + repo_metadata["recommended_package"] = library.recommended_package + if library.min_java_version: + repo_metadata["min_java_version"] = library.min_java_version + + # generate .repo-meta.json + json_file = ".repo-metadata.json" + # .repo-metadata.json is removed before generating the first version of + # a library. This check ensures no duplicated generation. + if not os.path.exists(f"{library_path}/{json_file}"): + with open(f"{library_path}/{json_file}", "w") as fp: + json.dump(repo_metadata, fp, indent=2) + + # generate .OwlBot-hermetic.yaml + owlbot_yaml_file = ".OwlBot-hermetic.yaml" + path_to_owlbot_yaml_file = ( + f"{library_path}/{owlbot_yaml_file}" + if config.is_monorepo() + else f"{library_path}/.github/{owlbot_yaml_file}" + ) + if not os.path.exists(path_to_owlbot_yaml_file): + render( + template_name="owlbot.yaml.monorepo.j2", + output_name=path_to_owlbot_yaml_file, + artifact_id=artifact_id, + proto_path=remove_version_from(proto_path), + module_name=repo_metadata["repo_short"], + api_shortname=library.api_shortname, + ) + + # generate owlbot.py + py_file = "owlbot.py" + template_excludes = [ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "java.header", + "license-checks.xml", + "renovate.json", + ".gitignore", + ] + if not os.path.exists(f"{library_path}/{py_file}"): + render( + template_name="owlbot.py.j2", + output_name=f"{library_path}/{py_file}", + should_include_templates=True, + template_excludes=template_excludes, + ) diff --git a/hermetic_build/library_generation/utils/utilities.sh b/hermetic_build/library_generation/utils/utilities.sh new file mode 100755 index 0000000000..a6112070a3 --- /dev/null +++ b/hermetic_build/library_generation/utils/utilities.sh @@ -0,0 +1,391 @@ +#!/usr/bin/env bash + +set -eo pipefail +utilities_script_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +# The $HOME variable is always set in the OS env as per POSIX specification. +GAPIC_GENERATOR_LOCATION="${HOME}/.library_generation/gapic-generator-java.jar" +JAVA_FORMATTER_LOCATION="${HOME}/.library_generation/google-java-format.jar" + +# Utility functions used in `generate_library.sh` and showcase generation. +extract_folder_name() { + local destination_path=$1 + local folder_name=${destination_path##*/} + echo "${folder_name}" +} + +remove_empty_files() { + local category=$1 + local destination_path=$2 + local file_num + find "${destination_path}/${category}-${folder_name}/src/main/java" -type f -size 0 | while read -r f; do rm -f "${f}"; done + # remove the directory if the directory has no files. + file_num=$(find "${destination_path}/${category}-${folder_name}" -type f | wc -l | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + if [[ "${file_num}" == 0 ]]; then + rm -rf "${destination_path}/${category}-${folder_name}" + fi + + if [ -d "${destination_path}/${category}-${folder_name}/src/main/java/samples" ]; then + mv "${destination_path}/${category}-${folder_name}/src/main/java/samples" "${destination_path}/${category}-${folder_name}" + fi +} + +# Move generated files to folders in destination_path. +mv_src_files() { + local category=$1 # one of gapic, proto, samples + local type=$2 # one of main, test + local destination_path=$3 + if [ "${category}" == "samples" ]; then + src_suffix="samples/snippets/generated/src/main/java" + folder_suffix="samples/snippets/generated" + mkdir -p "${destination_path}/${folder_suffix}" + cp -r "${destination_path}/java_gapic_srcjar/${src_suffix}"/* "${destination_path}/${folder_suffix}" + elif [ "${category}" == "proto" ]; then + src_suffix="${category}/src/${type}/java" + folder_suffix="${category}-${folder_name}/src/${type}" + else + src_suffix="src/${type}" + folder_suffix="${category}-${folder_name}/src" + fi + + if [ "${category}" == "samples" ]; then + return + fi + + mkdir -p "${destination_path}/${folder_suffix}" + cp -r "${destination_path}/java_gapic_srcjar/${src_suffix}" "${destination_path}/${folder_suffix}" + rm -r -f "${destination_path}/${folder_suffix}/java/META-INF" +} + +# unzip jar file +unzip_src_files() { + local category=$1 + local destination_path=$2 + local jar_file=java_${category}.jar + mkdir -p "${destination_path}/${category}-${folder_name}/src/main/java" + unzip -q -o "${destination_path}/${jar_file}" -d "${destination_path}/${category}-${folder_name}/src/main/java" + rm -r -f "${destination_path}/${category}-${folder_name}/src/main/java/META-INF" +} + +# get gapic options from .yaml and .json files from proto_path. +get_gapic_opts() { + local transport=$1 + local rest_numeric_enums=$2 + local gapic_yaml=$3 + local service_config=$4 + local service_yaml=$5 + if [ "${rest_numeric_enums}" == "true" ]; then + rest_numeric_enums="rest-numeric-enums" + else + rest_numeric_enums="" + fi + # If any of the gapic options is empty (default value), try to search for + # it in proto_path. + if [[ "${gapic_yaml}" == "" ]]; then + gapic_yaml=$(find "${proto_path}" -type f -name "*gapic.yaml") + fi + + if [[ "${service_config}" == "" ]]; then + service_config=$(find "${proto_path}" -type f -name "*service_config.json") + fi + + if [[ "${service_yaml}" == "" ]]; then + service_yaml=$(find "${proto_path}" -maxdepth 1 -type f \( -name "*.yaml" ! -name "*gapic*.yaml" \)) + fi + echo "transport=${transport},${rest_numeric_enums},grpc-service-config=${service_config},gapic-config=${gapic_yaml},api-service-config=${service_yaml}" +} + +remove_grpc_version() { + local destination_path=$1 + find "${destination_path}" -type f -name "*Grpc.java" -exec \ + sed -i.bak 's/value = \"by gRPC proto compiler.*/value = \"by gRPC proto compiler\",/g' {} \; -exec rm {}.bak \; +} + +# This function returns the version of the grpc plugin to generate the libraries. If +# DOCKER_GRPC_VERSION is set, this will be the version. Otherwise, the script +# will exit since this is a necessary env var +get_grpc_version() { + local grpc_version + if [[ -n "${DOCKER_GRPC_VERSION}" ]]; then + >&2 echo "Using grpc version baked into the container: ${DOCKER_GRPC_VERSION}" + echo "${DOCKER_GRPC_VERSION}" + return + else + >&2 echo "Cannot infer grpc version because DOCKER_GRPC_VERSION is not set" + exit 1 + fi +} + +# This function returns the version of protoc to generate the libraries. If +# DOCKER_PROTOC_VERSION is set, this will be the version. Otherwise, the script +# will exit since this is a necessary env var +get_protoc_version() { + local protoc_version + if [[ -n "${DOCKER_PROTOC_VERSION}" ]]; then + >&2 echo "Using protoc version baked into the container: ${DOCKER_PROTOC_VERSION}" + echo "${DOCKER_PROTOC_VERSION}" + return + else + >&2 echo "Cannot infer protoc version because DOCKER_PROTOC_VERSION is not set" + exit 1 + fi +} + +# Given the versions of the gapic generator, protoc and the protoc-grpc plugin, +# this function will download each one of the tools and create the environment +# variables "protoc_path" and "grpc_path" which are expected upstream. Note that +# if the specified versions of protoc and grpc match DOCKER_PROTOC_VERSION and +# DOCKER_GRPC_VERSION respectively, this function will instead set "protoc_path" +# and "grpc_path" to DOCKER_PROTOC_PATH and DOCKER_GRPC_PATH respectively (no +# download), since the docker image will have downloaded these tools beforehand. +# +# For the case of generator and formatter, no env var will be exported for the +# upstream flow. +# Instead, the jar must be located in the well-known location +# (${HOME}/.library_generation/). +# More information in `library_generation/DEVELOPMENT.md`. +download_tools() { + local protoc_version=$1 + local grpc_version=$2 + local os_architecture=$3 + pushd "${output_folder}" + + # the variable protoc_path is used in generate_library.sh. It is explicitly + # exported to make clear that it is used outside this utilities file. + if [[ "${DOCKER_PROTOC_VERSION}" == "${protoc_version}" ]]; then + # if the specified protoc_version matches the one baked in the docker + # container, we just point protoc_path to its location. + export protoc_path="${DOCKER_PROTOC_LOCATION}/protoc-${protoc_version}/bin" + else + export protoc_path=$(download_protoc "${protoc_version}" "${os_architecture}") + fi + + # similar case with grpc + if [[ "${DOCKER_GRPC_VERSION}" == "${grpc_version}" ]]; then + # if the specified grpc_version matches the one baked in the docker + # container, we just point grpc_path to its location. + export grpc_path="${DOCKER_GRPC_LOCATION}" + else + export grpc_path=$(download_grpc_plugin "${grpc_version}" "${os_architecture}") + fi + + # Here we check whether required tools is stored in the expected location. + # The docker image will prepare jar files in this location. + # This check is meant to ensure integrity of the downstream workflow. + error_if_not_exists "${GAPIC_GENERATOR_LOCATION}" + error_if_not_exists "${JAVA_FORMATTER_LOCATION}" + popd +} + +download_protoc() { + local protoc_version=$1 + local os_architecture=$2 + + local protoc_path="${output_folder}/protoc-${protoc_version}/bin" + + if [ ! -d "${protoc_path}" ]; then + # pull proto files and protoc from protobuf repository as maven central + # doesn't have proto files + download_from \ + "https://github.com/protocolbuffers/protobuf/releases/download/v${protoc_version}/protoc-${protoc_version}-${os_architecture}.zip" \ + "protoc-${protoc_version}.zip" \ + "GitHub" + unzip -o -q "protoc-${protoc_version}.zip" -d "protoc-${protoc_version}" + cp -r "protoc-${protoc_version}/include/google" . + rm "protoc-${protoc_version}.zip" + fi + echo "${protoc_path}" + +} + +download_grpc_plugin() { + local grpc_version=$1 + local os_architecture=$2 + grpc_filename="protoc-gen-grpc-java-${grpc_version}-${os_architecture}.exe" + if [ ! -f "${grpc_filename}" ]; then + # download protoc-gen-grpc-java plugin from Google maven central mirror. + download_from \ + "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/${grpc_version}/${grpc_filename}" \ + "${grpc_filename}" + chmod +x "${grpc_filename}" + fi + echo "$(pwd)/${grpc_filename}" +} + +download_from() { + local url=$1 + local save_as=$2 + local repo=$3 + # fail-fast, 30 seconds at most, retry 2 times + curl -LJ -o "${save_as}" --fail -m 30 --retry 2 "$url" || download_fail "${save_as}" "${repo}" +} + +# copies the specified file in $1 to $2 +# will return "true" if the copy was successful +copy_from() { + local local_repo=$1 + local save_as=$2 + copy_successful=$(cp "${local_repo}" "${save_as}" && echo "true" || echo "false") + echo "${copy_successful}" +} + +download_fail() { + local artifact=$1 + local repo=${2:-"maven central mirror"} + >&2 echo "Fail to download ${artifact} from ${repo} repository. Please install ${artifact} first if you want to use a non-published artifact." + exit 1 +} + +# gets the output folder where all sources and dependencies will be located. +get_output_folder() { + if [[ $(basename $(pwd)) != "output" ]]; then + echo "$(pwd)/output" + else + echo $(pwd) + fi +} + +detect_os_architecture() { + local os_type + local os_architecture + os_type=$(uname -sm) + case "${os_type}" in + *"Linux x86_64"*) + os_architecture="linux-x86_64" + ;; + *"Darwin x86_64"*) + os_architecture="osx-x86_64" + ;; + *) + os_architecture="osx-aarch_64" + ;; + esac + echo "${os_architecture}" +} + + +# copies $1 as a folder as $2 only if $1 exists +copy_directory_if_exists() { + local base_folder=$1 + local folder_prefix=$2 + local destination_folder=$3 + if [ ! -d "${base_folder}" ]; then + return + fi + pushd "${base_folder}" + if [[ $(find . -maxdepth 1 -type d -name "${folder_prefix}*" | wc -l ) -gt 0 ]]; then + cp -r ${base_folder}/${folder_prefix}* "${destination_folder}" + fi + popd # base_folder +} + +# computes proto_path from a given folder of GAPIC sources +# It will inspect the proto library to compute the path +get_proto_path_from_preprocessed_sources() { + set -e + local sources=$1 + pushd "${sources}" > /dev/null + local proto_library=$(find . -maxdepth 1 -type d -name 'proto-*' | sed 's/\.\///') + local found_libraries=$(echo "${proto_library}" | wc -l) + if [[ -z ${proto_library} ]]; then + echo "no proto libraries found in the supplied sources path" + exit 1 + elif [ ${found_libraries} -gt 1 ]; then + echo "more than one proto library found in the supplied sources path" + echo "cannot decide for a service version" + exit 1 + fi + pushd "$(pwd)/${proto_library}/src/main/proto" > /dev/null + local result=$(find . -type f -name '*.proto' | head -n 1 | xargs dirname | sed 's/\.\///') + popd > /dev/null # proto_library + popd > /dev/null # sources + echo "${result}" +} + +# for a pre-processed library stored in $preprocessed_sources_path, a folder +# tree is built on $target_folder so it looks like a googleapis-gen folder and +# is therefore consumable by OwlBot CLI +build_owlbot_cli_source_folder() { + local postprocessing_target=$1 + local target_folder=$2 + local preprocessed_sources_path=$3 + local proto_path=$4 + if [[ -z "${proto_path}" ]]; then + proto_path=$(get_proto_path_from_preprocessed_sources "${preprocessed_sources_path}") + fi + owlbot_staging_folder="${postprocessing_target}/owl-bot-staging" + mkdir -p "${owlbot_staging_folder}" + + # By default (thanks to generation templates), .OwlBot-hermetic.yaml `deep-copy` section + # references a wildcard pattern matching a folder + # ending with `-java` at the leaf of proto_path. We then use a generated-java + # folder that will be picked up by copy-code + mkdir -p "${target_folder}/${proto_path}/generated-java" + copy_directory_if_exists "${preprocessed_sources_path}" "proto" \ + "${target_folder}/${proto_path}/generated-java/proto-google-cloud-library" + copy_directory_if_exists "${preprocessed_sources_path}" "grpc" \ + "${target_folder}/${proto_path}/generated-java/grpc-google-cloud-library" + copy_directory_if_exists "${preprocessed_sources_path}" "gapic" \ + "${target_folder}/${proto_path}/generated-java/gapic-google-cloud-library" + copy_directory_if_exists "${preprocessed_sources_path}" "samples" \ + "${target_folder}/${proto_path}/generated-java/samples" + pushd "${target_folder}" + # create an empty commit so owl-bot-copy can process this as a repo + # (it cannot process non-git-repositories) + git init + git commit --allow-empty -m 'empty commit' + popd # target_folder +} + +# Convenience function to clone only the necessary folders from a git repository +sparse_clone() { + repo_url=$1 + paths=$2 + commitish=$3 + clone_dir=$(basename "${repo_url%.*}") + rm -rf "${clone_dir}" + git clone -n --depth=1 --no-single-branch --filter=tree:0 "${repo_url}" + pushd "${clone_dir}" + if [ -n "${commitish}" ]; then + git checkout "${commitish}" + fi + git sparse-checkout set --no-cone ${paths} + git checkout + popd +} + +# calls a function in utilities.py. THe first argument is the function name, the +# rest of the arguments are the positional arguments to such function +py_util() { + python3 "${utilities_script_dir}/utilities.py" "$@" +} + +download_googleapis_files_and_folders() { + local output_folder=$1 + local googleapis_commitish=$2 + # checkout the master branch of googleapis/google (proto files) and WORKSPACE + echo "Checking out googlapis repository..." + # sparse_clone will remove folder contents first, so we have to checkout googleapis + # only once. + sparse_clone https://github.com/googleapis/googleapis.git "google grafeas" "${googleapis_commitish}" + pushd googleapis + cp -r google "${output_folder}" + cp -r grafeas "${output_folder}" +} + +get_gapic_generator_location() { + echo "${GAPIC_GENERATOR_LOCATION}" +} + +get_java_formatter_location() { + echo "${JAVA_FORMATTER_LOCATION}" +} + +error_if_not_exists() { + local required_tool=$1 + if [[ ! -f "${required_tool}" ]]; then + >&2 echo "File ${required_tool} not found in the filesystem. " + >&2 echo "Please configure your environment and store the " + >&2 echo "required tools in this location." + exit 1 + fi +} diff --git a/library_generation/DEVELOPMENT.md b/library_generation/DEVELOPMENT.md deleted file mode 100644 index f32b544e1e..0000000000 --- a/library_generation/DEVELOPMENT.md +++ /dev/null @@ -1,207 +0,0 @@ -> [!IMPORTANT] -> All examples assume you are inside the `library_generation` folder. - - -# Linting - -When contributing, ensure your changes to python code have a valid format. - -``` -python -m pip install black -black . -``` - -# Running the integration tests - -The integration tests build the docker image declared in -`.cloudbuild/library_generation/library_generation.Dockerfile`, pull GAPIC -repositories, generate the libraries and compares the results with the source -code declared in a "golden branch" of the repo. - -It requires docker and python 3.x to be installed. - -``` -python -m pip install . -python -m pip install -r requirements.txt -python -m unittest test/integration_tests.py -``` - -# Running the unit tests - -The unit tests of the hermetic build scripts are contained in several scripts, -corresponding to a specific component. -Every unit test script ends with `unit_tests.py`. -To avoid them specifying them individually, we can use the following command: - -```bash -python -m unittest discover -s test/ -p "*unit_tests.py" -``` - -> [!NOTE] -> The output of this command may look erratic during the first 30 seconds. -> This is normal. After the tests are done, an "OK" message should be shown. - -# Running the scripts in your local environment - -Although the scripts are designed to be run in a Docker container, you can also -run them directly. -This section explains how to run the entrypoint script -(`library_generation/cli/entry_point.py`). - -## Assumptions made by the scripts -### The Hermetic Build's well-known folder -Located in `${HOME}/.library_generation`, this folder is assumed by the scripts -to contain certain tools. - -Developers must make sure this folder is properly configured before running the -scripts locally. -Note that this relies on the `HOME` en var which is always defined as per -[POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html). - -#### Put the gapic-generator-java jar in its well-known location - -Run `cd sdk-platform-java && mvn install -DskipTests -Dclirr.skip --Dcheckstyle.skip`. -This will generate a jar located in -`~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar` - -Then `mv` the jar into the well-known location of the jar. -The generation scripts will assume the jar is there. - -```shell -mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar" -``` - -#### Put the java formatter jar in its well-known location - -Download google-java-format-{version}-all-deps.jar from [Maven Central](https://central.sonatype.com/artifact/com.google.googlejavaformat/google-java-format) -or [GitHub releases](https://github.com/google/google-java-format/releases). -Then `mv` the jar into the well-known location of the jar. -The generation scripts will assume the jar is there. - -```shell -mv /path/to/jar "${HOME}/.library_generation/google-java-format.jar" -``` - -## Installing prerequisites - -In order to run the generation scripts directly, there are a few tools we -need to install beforehand. - -### Install the owl-bot CLI - -Requires node.js to be installed. -Check this [installation guide](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script) -for NVM, Node.js's version manager. - -After you install it, you can install the owl-bot CLI with the following -commands: -```bash -git clone https://github.com/googleapis/repo-automation-bots -cd repo-automation-bots/packages/owl-bot -npm i && npm run compile && npm link -owl-bot copy-code --version -``` - -The key step is `npm link`, which will make the command available in you current -shell session. - - -## Running the script -The entrypoint script (`library_generation/cli/entry_point.py`) allows you to -generate a GAPIC repository with a given api definition (proto, service yaml). - -### Download the api definition -For example, googleapis -``` -git clone https://github.com/googleapis/googleapis -export api_definitions_path="$(pwd)/googleapis" -``` - -### Download the repo -For example, google-cloud-java -``` -git clone https://github.com/googleapis/google-cloud-java -export path_to_repo="$(pwd)/google-cloud-java" -``` - -### Install the scripts -``` -python -m pip install . -``` - -### Run the script -``` -python cli/entry_point.py generate \ - --repository-path="${path_to_repo}" \ - --api-definitions-path="${api_definitions_path}" -``` - - -# Running the scripts using the docker container image -This is convenient in order to avoid installing the dependencies manually. - -> [!IMPORTANT] -> From now, the examples assume you are in the root of your sdk-platform-java -> folder. - -## Build the docker image -```bash -docker build --file .cloudbuild/library_generation/library_generation.Dockerfile --iidfile image-id . -``` - -This will create an `image-id` file at the root of the repo with the hash ID of -the image. - -## Run the docker image -The docker image will perform changes on its internal `/workspace` folder, -to which you need to map a folder on your host machine (i.e. map your downloaded -repo to this folder). - -To run the docker container on the google-cloud-java repo, you must run: -```bash -docker run \ - -u "$(id -u)":"$(id -g)" \ - -v /path/to/google-cloud-java:/workspace \ - -v /path/to/api-definition:/workspace/apis \ - $(cat image-id) \ - --api-definitions-path=/workspace/apis -``` - - * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating - yourself. This avoids folder ownership changes since it runs as root by - default. - * `-v /path/to/google-cloud-java:/workspace` maps the host machine's - google-cloud-java folder to the /workspace folder. - The image is configured to perform changes in this directory. - * `-v /path/to/api-definition:/workspace/apis` maps the host machine's - api-definition folder to /workspace/apis folder. - * `$(cat image-id)` obtains the image ID created in the build step. - * `--api-definitions-path=/workspace/apis` set the API definition path to - `/workspace/apis`. - -## Debug the created containers -If you are working on changing the way the containers are created, you may want -to inspect the containers to check the setup. -It would be convenient in such case to have a text editor/viewer available. -You can achieve this by modifying the Dockerfile as follows: - -```docker -# install OS tools -RUN apt-get update && apt-get install -y \ - unzip openjdk-17-jdk rsync maven jq less vim \ - && apt-get clean -``` - -We add `less` and `vim` as text tools for further inspection. - -You can also run a shell in a new container by running: - -```bash -docker run \ - --rm -it \ - -u $(id -u):$(id -g) \ - -v /path/to/google-cloud-java:/workspace \ - --entrypoint="bash" \ - $(cat image-id) -``` diff --git a/library_generation/README.md b/library_generation/README.md deleted file mode 100644 index 518d588f9a..0000000000 --- a/library_generation/README.md +++ /dev/null @@ -1,310 +0,0 @@ -# Generate a repository containing GAPIC Client Libraries - -The script, `entry_point.py`, allows you to generate a repository containing -GAPIC client libraries (a monorepo, for example, google-cloud-java) from a -configuration file. - -## Environment - -- OS: Linux -- Java runtime environment (8 or above) -- Python (3.11.6 or above) -- Docker -- Git - -## Prerequisite - -In order to generate a version for each library, a versions.txt has to exist -in `repository_path`. -Please refer to [Repository path](#repository-path--repositorypath---optional) for more information. - -## Parameters to generate a repository using `entry_point.py` - -### Baseline generation configuration yaml (`baseline_generation_config`) - -An absolute or relative path to a generation_config.yaml. -This config file is used for computing changed libraries, not library -generation. - -### Current generation configuration yaml (`current_generation_config`) - -An absolute or relative path to a configuration file containing parameters to -generate the repository. -Please refer [Configuration to generate a repository](#configuration-to-generate-a-repository) -for more information. - -### Repository path (`repository_path`), optional - -The path to where the generated repository goes. - -The default value is the current working directory when running the script. -For example, `cd google-cloud-java && python entry_point.py ...` without -specifying the `--repository_path` option will modify the `google-cloud-java` -repository the user `cd`'d into. - -Note that versions.txt has to exist in `repository_path` in order to generate -right version for each library. -Please refer [here](go/java-client-releasing#versionstxt-manifest) for more info -of versions.txt. - -### Api definitions path (`api_definitions_path`), optional - -The path to where the api definition (proto, service yaml) resides. - -The default value is the current working directory when running the script. - -Note that you need not only the protos defined the service, but also the transitive -dependencies of those protos. -Any missing dependencies will cause `File not found` error. - -For example, if your service is defined in `example_service.proto` and it imports -`google/api/annotations.proto`, you need the `annotations.proto` resides in a -folder that has the exact structure of the import statement (`google/api` in this -case), and set `api_definitions_path` to the path contains the root folder (`google` -in this case). - -## Output of `entry_point.py` - -### GAPIC libraries - -For each module (e.g. `google-cloud-java/java-asset`), the following files/folders -will be created/modified: - -| Name | Notes | -|:------------------------------------|:-------------------------------------------------------------------------| -| google-*/ | Source code generated by gapic-generator-java | -| google-*/pom.xml | Only be generated if it does not exist | -| grpc-*/ | Source code generated by grpc generator, one per each version | -| grpc-*/pom.xml | Only be generated if it does not exist | -| proto-*/ | Source code generated by Protobuf default compiler, one per each version | -| proto-*/pom.xml | Only be generated if it does not exist | -| samples/snippets/generated/ | Only be generated if `include_samples` is set to true | -| google-*-bom/pom.xml | Library BOM, only be generated if it does not exist | -| pom.xml | Library parent BOM, only be generated if it does not exist | -| .repo-metadata.json | Always generated from inputs | -| .OwlBot-hermetic.yaml | Only be generated from a template if it does not exist | -| owlbot.py | Only be generated from a template if it does not exist | -| README.md | Always generated from inputs | -| gapic-libraries-bom/pom.xml | Always generated from inputs | -| pom.xml (repo root dir) | Always generated from inputs | -| versions.txt | New entries will be added if they don’t exist | - -## Configuration to generate a repository - -There are three levels of parameters in the configuration: repository level, -library level and GAPIC level. - -### Repository level parameters - -The repository level parameters define the version of API definition (proto) -and tools. -They are shared by library level parameters. - -| Name | Required | Notes | -|:------------------------|:--------:|:---------------------------------------------| -| gapic_generator_version | No | set through env variable if not specified | -| protoc_version | No | inferred from the generator if not specified | -| grpc_version | No | inferred from the generator if not specified | -| googleapis_commitish | Yes | | -| libraries_bom_version | No | empty string if not specified | - -### Library level parameters - -The library level parameters define how to generate a (multi-versions) GAPIC -library. -They are shared by all GAPICs of a library. - -| Name | Required | Notes | -|:----------------------|:--------:|:------------------------------------------------------------------------------------------------------------------------------------------| -| api_shortname | Yes | | -| api_description | Yes | | -| name_pretty | Yes | | -| product_docs | Yes | | -| library_type | No | `GAPIC_AUTO` if not specified | -| release_level | No | `preview` if not specified | -| api_id | No | `{api_shortname}.googleapis.com` if not specified | -| api_reference | No | | -| codeowner_team | No | | -| client_documentation | No | | -| distribution_name | No | `{group_id}:google-{cloud_prefix}{library_name}` if not specified | -| excluded_poms | No | | -| excluded_dependencies | No | | -| googleapis_commitish | No | use repository level `googleapis_commitish` if not specified. | -| group_id | No | `com.google.cloud` if not specified | -| issue_tracker | No | | -| library_name | No | `api_shortname` is not specified. This value should be unique among all libraries. | -| rest_documentation | No | | -| rpc_documentation | No | | -| cloud_api | No | `true` if not specified | -| requires-billing | No | `true` if not specified | -| transport | No | must be one of `grpc`, `rest` or `both`. This value would only be used for generating .repo-metadata.json and relevant sections in README | - - -Note that `cloud_prefix` is `cloud-` if `cloud_api` is `true`; empty otherwise. - -### GAPIC level parameters - -The GAPIC level parameters define how to generate a GAPIC library. - -| Name | Required | Notes | -|:-----------|:--------:|:------------------------------------------| -| proto_path | Yes | versioned proto_path starts with `google` | - -### An example of generation configuration - -```yaml -gapic_generator_version: 2.34.0 -protoc_version: 25.2 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." - release_level: "stable" - library_name: "apigee-connect" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 - - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 -``` - -# Local Environment Setup before running `entry_point.py` - -1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. -The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case). -2. Assuming the virtual environment is installed under `sdk-platform-java`. -Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation` - - ```bash - python -m pip install --require-hashes -r library_generation/requirements.txt - ``` - -3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere - ```bash - python -m pip install library_generation/ - ``` - -4. Download api definition to a local directory - -## An example to generate a repository using `entry_point.py` - -```bash -python library_generation/entry_point.py generate \ ---baseline-generation-config-path=/path/to/baseline_config_file \ ---current-generation-config-path=/path/to/current_config_file \ ---repository-path=path/to/repository \ ---api-definitions-path=path/to/api_definition -``` -If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration) -shown above, the repository structure is: -``` -$repository_path -|_gapic-libraries-bom -| |_pom.xml -|_java-apigee-connect -| |_google-cloud-apigee-connect -| | |_src -| | |_pom.xml -| |_google-cloud-apigee-connect-bom -| | |_pom.xml -| |_grpc-google-cloud-apigee-connect-v1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-apigee-connect-v1 -| | |_src -| | |_pom.xml -| |_samples -| | |_snippets -| | | |_generated -| |_.OwlBot-hermetic.yaml -| |_.repo-metadata.json -| |_owlbot.py -| |_pom.xml -| |_README.md -|_java-asset -| |_google-cloud-asset -| | |_src -| | |_pom.xml -| |_google-cloud-asset-bom -| | |_pom.xml -| |_grpc-google-cloud-asset-v1 -| | |_src -| | |_pom.xml -| |_grpc-google-cloud-asset-v1p1beta1 -| | |_src -| | |_pom.xml -| |_grpc-google-cloud-asset-v1p2beta1 -| | |_src -| | |_pom.xml -| |_grpc-google-cloud-asset-v1p5beta1 -| | |_src -| | |_pom.xml -| |_grpc-google-cloud-asset-v1p7beta1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-asset-v1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-asset-v1p1beta1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-asset-v1p2beta1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-asset-v1p5beta1 -| | |_src -| | |_pom.xml -| |_proto-google-cloud-asset-v1p7beta1 -| | |_src -| | |_pom.xml -| |_samples -| | |_snippets -| | | |_generated -| |_.OwlBot-hermetic.yaml -| |_.repo-metadata.json -| |_owlbot.py -| |_pom.xml -| |_README.md -|_pom.xml -|_versions.txt -``` - -# Owlbot Java Postprocessor - -We have transferred the -[implementation](https://github.com/googleapis/synthtool/tree/59fe44fde9866a26e7ee4e4450fd79f67f8cf599/docker/owlbot/java) -of Java Owlbot Postprocessor into `sdk-platform-java/library_generation`. The -implementation in synthtool is still valid and used by other services, so we -have two versions during a transition period. - -## Reflecting changes in synthtool/docker/owlbot/java into this repository -The transfer was not a verbatim copy, it rather had modifications: - * `format-source.sh` was replaced by a call to `mvn fmt:format` - * `entrypoint.sh` was modified to have input arguments and slightly modified - the way the helper scripts are called - * Other helper scripts were modified to have input arguments. - * `fix_poms.py` modified the way the monorepo is detected - -All these modifications imply that whenever we want to reflect a change from the -original owlbot in synthtool we may be better off modifying the affected source -files one by one. The mapping is from -[`synthtool/docker/owlbot/java`](https://github.com/googleapis/synthtool/tree/59fe44fde9866a26e7ee4e4450fd79f67f8cf599/docker/owlbot/java) -to -[`sdk-platform-java/library_generation/owlbot`](https://github.com/googleapis/sdk-platform-java/tree/move-java-owlbot/library_generation/owlbot) From 55780477b098b91df6e96904a32d490c4f0f947e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 13:46:43 -0400 Subject: [PATCH 18/69] add release_note_generation module --- .../cli/generate_release_note.py | 98 +++++++++ .../commit_message_formatter.py | 127 ++++++++++++ .../generate_pr_description.py | 191 ++++++++++++++++++ .../release_note_generation/setup.py | 7 +- .../cli/generate_release_note_unit_tests.py | 2 +- .../commit_message_formatter_unit_tests.py | 9 +- .../generate_pr_description_unit_tests.py | 9 +- .../goldens/.OwlBot-hermetic-golden.yaml | 35 ++++ ...repo-metadata-custom-transport-golden.json | 16 ++ .../.repo-metadata-monorepo-golden.json | 20 ++ .../.repo-metadata-non-monorepo-golden.json | 21 ++ .../.repo-metadata-proto-only-golden.json | 19 ++ .../tests/resources/goldens/owlbot-golden.py | 36 ++++ .../goldens/pr_description-golden.txt | 17 ++ ...qualified_commit_pr_description-golden.txt | 10 + .../repo_level_only_pr_description-golden.txt | 5 + .../empty_gen_config.yaml | 0 17 files changed, 609 insertions(+), 13 deletions(-) create mode 100644 hermetic_build/release_note_generation/cli/generate_release_note.py create mode 100644 hermetic_build/release_note_generation/commit_message_formatter.py create mode 100755 hermetic_build/release_note_generation/generate_pr_description.py create mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml create mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json create mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json create mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json create mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json create mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py create mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/pr_description-golden.txt create mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt create mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt create mode 100644 hermetic_build/release_note_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml diff --git a/hermetic_build/release_note_generation/cli/generate_release_note.py b/hermetic_build/release_note_generation/cli/generate_release_note.py new file mode 100644 index 0000000000..adc93d9ea7 --- /dev/null +++ b/hermetic_build/release_note_generation/cli/generate_release_note.py @@ -0,0 +1,98 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from typing import Optional +import click as click +from release_note_generation.generate_pr_description import generate_pr_descriptions +from common.model.generation_config import from_yaml +from library_generation.utils.generation_config_comparator import compare_config + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--baseline-generation-config-path", + required=False, + default=None, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml. + This config file is used for commit history generation, not library + generation. + """, +) +@click.option( + "--current-generation-config-path", + required=False, + default=None, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml that contains the + metadata about library generation. + """, +) +@click.option( + "--repository-path", + type=str, + default=".", + show_default=True, + help=""" + The repository path to which the generated files will be sent. + If not specified, the repository will be generated to the current working + directory. + """, +) +def generate( + baseline_generation_config_path: Optional[str], + current_generation_config_path: Optional[str], + repository_path: str, +) -> None: + if ( + baseline_generation_config_path is None + or current_generation_config_path is None + ): + print( + "One of the generation configs is not specified, do not generate " + "the description." + ) + return + baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) + current_generation_config_path = os.path.abspath(current_generation_config_path) + if not ( + os.path.isfile(baseline_generation_config_path) + and os.path.isfile(current_generation_config_path) + ): + print( + "One of the generation configs does not exist, do not generate " + "the description." + ) + return + config_change = compare_config( + baseline_config=from_yaml(baseline_generation_config_path), + current_config=from_yaml(current_generation_config_path), + ) + generate_pr_descriptions( + config_change=config_change, + description_path=os.path.abspath(repository_path), + ) + + +if __name__ == "__main__": + main() diff --git a/hermetic_build/release_note_generation/commit_message_formatter.py b/hermetic_build/release_note_generation/commit_message_formatter.py new file mode 100644 index 0000000000..48c060c571 --- /dev/null +++ b/hermetic_build/release_note_generation/commit_message_formatter.py @@ -0,0 +1,127 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re +from git import Commit + +from library_generation.model.config_change import ConfigChange, ChangeType +from common.model.generation_config import ( + GAPIC_GENERATOR_VERSION, + LIBRARIES_BOM_VERSION, +) + +PARAM_TO_COMMIT_MESSAGE = { + GAPIC_GENERATOR_VERSION: "fix(deps): update the Java code generator (gapic-generator-java) to", + LIBRARIES_BOM_VERSION: "chore: update the libraries_bom version to", +} + + +def format_commit_message(commits: dict[Commit, str], is_monorepo: bool) -> list[str]: + """ + Format commit messages. Add library_name to conventional commit messages if + is_monorepo is True; otherwise no op. + + :param commits: a mapping from commit to library_name. + :param is_monorepo: whether it's monorepo or not. + :return: formatted commit messages. + """ + all_commits = [] + # please see go/java-client-releasing#conventional-commit-messages + # for conventional commit. + type_regex = re.compile(r"(feat|fix|docs|deps|test|samples|chore)!?:.*") + for commit, library_name in commits.items(): + # a commit message may contain multiple lines, we need to + # add library_name for each line. + messages = [] + for message_line in commit.message.split("\n"): + # add library name to a conventional commit message; + # otherwise no op. + if type_regex.search(message_line): + commit_type, _, summary = message_line.partition(":") + formatted_message = ( + f"{commit_type}: [{library_name}]{str(summary).rstrip()}" + if is_monorepo + else f"{commit_type}:{str(summary).rstrip()}" + ) + messages.append(formatted_message) + else: + messages.append(message_line) + all_commits.extend(wrap_googleapis_commit(commit, messages)) + return all_commits + + +def format_repo_level_change(config_change: ConfigChange) -> list[str]: + """ + Format commit messages regarding repo-level changes. + + :param config_change: + :return: commit messages regarding repo-level changes. + """ + messages = [] + for repo_level_change in config_change.change_to_libraries.get( + ChangeType.REPO_LEVEL_CHANGE, [] + ): + message = f"chore: update repo-level parameter {repo_level_change.changed_param} to {repo_level_change.current_value}" + if repo_level_change.changed_param in PARAM_TO_COMMIT_MESSAGE: + message = f"{PARAM_TO_COMMIT_MESSAGE.get(repo_level_change.changed_param)} {repo_level_change.current_value}" + messages.extend(__wrap_nested_commit([message])) + return messages + + +def wrap_googleapis_commit(commit: Commit, messages: list[str]) -> list[str]: + """ + Wrap message between `BEGIN_NESTED_COMMIT` and `BEGIN_NESTED_COMMIT`. + + :param commit: a GitHub commit. + :param messages: a (multi-line) commit message, one line per item. + :return: wrapped messages. + """ + messages.append(f"Source Link: {commit_link(commit)}") + return __wrap_nested_commit(messages) + + +def wrap_override_commit(messages: list[str]) -> list[str]: + """ + Wrap message between `BEGIN_COMMIT_OVERRIDE` and `END_COMMIT_OVERRIDE`. + + :param messages: a (multi-line) commit message, one line per item. + :return: wrapped messages. + """ + result = ["BEGIN_COMMIT_OVERRIDE"] + result.extend(messages) + result.append("END_COMMIT_OVERRIDE") + return result + + +def commit_link(commit: Commit) -> str: + """ + Create a link to the commit in Markdown format. + + :param commit: a GitHub commit. + :return: a link in Markdown format. + """ + short_sha = commit.hexsha[:7] + return f"[googleapis/googleapis@{short_sha}](https://github.com/googleapis/googleapis/commit/{commit.hexsha})" + + +def __wrap_nested_commit(messages: list[str]) -> list[str]: + """ + Wrap message between `BEGIN_NESTED_COMMIT` and `BEGIN_NESTED_COMMIT`. + + :param messages: a (multi-line) commit message, one line per item. + :return: wrapped messages. + """ + result = ["BEGIN_NESTED_COMMIT"] + result.extend(messages) + result.append("END_NESTED_COMMIT") + return result diff --git a/hermetic_build/release_note_generation/generate_pr_description.py b/hermetic_build/release_note_generation/generate_pr_description.py new file mode 100755 index 0000000000..64119a1283 --- /dev/null +++ b/hermetic_build/release_note_generation/generate_pr_description.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import calendar +import os +import shutil +from typing import Dict +from git import Commit, Repo + +from library_generation.model.config_change import ConfigChange +from library_generation.utils.proto_path_utils import find_versioned_proto_path +from release_note_generation.commit_message_formatter import ( + format_commit_message, + format_repo_level_change, + commit_link, + wrap_override_commit +) + +EMPTY_MESSAGE = "" + + +def generate_pr_descriptions( + config_change: ConfigChange, + description_path: str, + repo_url: str = "https://github.com/googleapis/googleapis.git", +) -> None: + """ + Generate pull request description from configuration comparison result. + + The pull request description contains repo-level changes, if applicable, + and googleapis commit from baseline config (exclusive) to current config + (inclusive). + + The pull request description will be generated into + description_path/pr_description.txt. + + No pr_description.txt will be generated if no changes in the configurations. + + :param config_change: a ConfigChange object, containing changes in baseline + and current generation configurations. + :param description_path: the path to which the pull request description + file goes. + :param repo_url: the GitHub repository from which retrieves the commit + history. + """ + repo_level_message = format_repo_level_change(config_change) + paths = config_change.current_config.get_proto_path_to_library_name() + description = get_repo_level_commit_messages( + repo_url=repo_url, + current_commit_sha=config_change.current_config.googleapis_commitish, + baseline_commit_sha=config_change.baseline_config.googleapis_commitish, + paths=paths, + is_monorepo=config_change.current_config.is_monorepo(), + repo_level_message=repo_level_message, + ) + + if description == EMPTY_MESSAGE: + print("Empty commit messages, skip creating pull request description.") + return + + description_file = f"{description_path}/pr_description.txt" + print(f"Writing pull request description to {description_file}") + with open(description_file, "w+") as f: + f.write(description) + + +def get_repo_level_commit_messages( + repo_url: str, + current_commit_sha: str, + baseline_commit_sha: str, + paths: Dict[str, str], + is_monorepo: bool, + repo_level_message: list[str], +) -> str: + """ + Combine commit messages of a repository from latest_commit to + baseline_commit. Only commits which change files in a pre-defined + file paths will be considered. + Note that baseline_commit should be an ancestor of or the same as + latest_commit. + + :param repo_url: the url of the repository. + :param current_commit_sha: the newest commit to be considered in + selecting commit message. + :param baseline_commit_sha: the oldest commit to be considered in + selecting commit message. This commit should be an ancestor of + :param paths: a mapping from file paths to library_name. + :param is_monorepo: whether to generate commit messages in a monorepo. + :param repo_level_message: commit messages regarding repo-level changes. + :return: commit messages. + :raise ValueError: if current_commit is older than or equal to + baseline_commit. + """ + tmp_dir = "/tmp/repo" + shutil.rmtree(tmp_dir, ignore_errors=True) + os.mkdir(tmp_dir) + repo = Repo.clone_from(repo_url, tmp_dir) + current_commit = repo.commit(current_commit_sha) + baseline_commit = repo.commit(baseline_commit_sha) + current_commit_time = __get_commit_timestamp(current_commit) + baseline_commit_time = __get_commit_timestamp(baseline_commit) + if current_commit_time < baseline_commit_time: + raise ValueError( + f"current_commit ({current_commit_sha[:7]}, committed on " + f"{current_commit_time}) should be newer than or equal to " + f"baseline_commit ({baseline_commit_sha[:7]}, committed on " + f"{baseline_commit_time})." + ) + qualified_commits = {} + commit = current_commit + while str(commit.hexsha) != baseline_commit_sha: + commit_and_name = __filter_qualified_commit(paths=paths, commit=commit) + if commit_and_name != (): + qualified_commits[commit_and_name[0]] = commit_and_name[1] + commit_parents = commit.parents + if len(commit_parents) == 0: + break + commit = commit_parents[0] + shutil.rmtree(tmp_dir, ignore_errors=True) + + return __combine_commit_messages( + current_commit=current_commit, + baseline_commit=baseline_commit, + commits=qualified_commits, + is_monorepo=is_monorepo, + repo_level_message=repo_level_message, + ) + + +def __filter_qualified_commit(paths: Dict[str, str], commit: Commit) -> (Commit, str): + """ + Returns a tuple of a commit and libray_name. + A qualified commit means at least one file, excluding BUILD.bazel, changes + in that commit is within the versioned proto_path in paths. + + :param paths: a mapping from versioned proto_path to library_name. + :param commit: a commit under consideration. + :return: a tuple of a commit and library_name if the commit is + qualified; otherwise an empty tuple. + """ + for file in commit.stats.files.keys(): + versioned_proto_path = find_versioned_proto_path(file) + if versioned_proto_path in paths and (not file.endswith("BUILD.bazel")): + return commit, paths[versioned_proto_path] + return () + + +def __combine_commit_messages( + current_commit: Commit, + baseline_commit: Commit, + commits: Dict[Commit, str], + is_monorepo: bool, + repo_level_message: list[str], +) -> str: + description = [] + if current_commit != baseline_commit: + description.append( + f"This pull request is generated with proto changes between " + f"{commit_link(baseline_commit)} (exclusive) " + f"and {commit_link(current_commit)} (inclusive).\n", + ) + commit_message = repo_level_message + commit_message.extend( + format_commit_message(commits=commits, is_monorepo=is_monorepo) + ) + if len(commit_message) == 0: + return EMPTY_MESSAGE + description.extend(wrap_override_commit(commit_message)) + return "\n".join(description) + + +def __get_commit_timestamp(commit: Commit) -> int: + """ + # Convert datetime to UTC timestamp. For more info: + # https://stackoverflow.com/questions/5067218/get-utc-timestamp-in-python-with-datetime + + :param commit: a Commit object + :return: the timestamp of the commit + """ + return calendar.timegm(commit.committed_datetime.utctimetuple()) diff --git a/hermetic_build/release_note_generation/setup.py b/hermetic_build/release_note_generation/setup.py index 1adf08151a..81628e9120 100755 --- a/hermetic_build/release_note_generation/setup.py +++ b/hermetic_build/release_note_generation/setup.py @@ -5,13 +5,16 @@ from setuptools import setup setup( - name="common", + name="release_note_generation", version="0.1", python_requires=">=3.12", package_dir={ - "common": ".", + "release_note_generation": ".", }, install_requires=[ + "click==8.1.7", + "common==0.1", # local package + "GitPython==3.1.43", "PyYAML==6.0.2", ], ) diff --git a/hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py b/hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py index be0d6a4fed..e1ee1e31e8 100644 --- a/hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py +++ b/hermetic_build/release_note_generation/tests/cli/generate_release_note_unit_tests.py @@ -14,7 +14,7 @@ import os import unittest from click.testing import CliRunner -from library_generation.cli.generate_release_note import generate +from release_note_generation.cli.generate_release_note import generate script_dir = os.path.dirname(os.path.realpath(__file__)) resource_dir = os.path.join(script_dir, "..", "resources", "test_generate_release_note") diff --git a/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py b/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py index 16e3fffdfc..8b4fccf2a7 100644 --- a/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py +++ b/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py @@ -13,20 +13,19 @@ # limitations under the License. import unittest from unittest.mock import patch - from library_generation.model.config_change import ( ConfigChange, ChangeType, LibraryChange, ) -from library_generation.model.generation_config import GenerationConfig -from library_generation.utils.commit_message_formatter import ( +from common.model.generation_config import GenerationConfig +from release_note_generation.commit_message_formatter import ( format_commit_message, commit_link, format_repo_level_change, + wrap_googleapis_commit, + wrap_override_commit ) -from library_generation.utils.commit_message_formatter import wrap_googleapis_commit -from library_generation.utils.commit_message_formatter import wrap_override_commit gen_config = GenerationConfig( gapic_generator_version="1.2.3", googleapis_commitish="123abc", libraries=[] diff --git a/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py b/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py index 9adbe71277..7c6f0db9ac 100644 --- a/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py +++ b/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py @@ -14,8 +14,7 @@ import os import unittest from filecmp import cmp - -from library_generation.generate_pr_description import ( +from release_note_generation.generate_pr_description import ( get_repo_level_commit_messages, generate_pr_descriptions, ) @@ -24,9 +23,9 @@ ChangeType, LibraryChange, ) -from library_generation.model.gapic_config import GapicConfig -from library_generation.model.generation_config import GenerationConfig -from library_generation.model.library_config import LibraryConfig +from common.model.gapic_config import GapicConfig +from common.model.generation_config import GenerationConfig +from common.model.library_config import LibraryConfig script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, "resources", "goldens") diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml b/hermetic_build/release_note_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml new file mode 100644 index 0000000000..225b4620bf --- /dev/null +++ b/hermetic_build/release_note_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml @@ -0,0 +1,35 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +deep-remove-regex: +- "/java-bare-metal-solution/grpc-google-.*/src" +- "/java-bare-metal-solution/proto-google-.*/src" +- "/java-bare-metal-solution/google-.*/src" +- "/java-bare-metal-solution/samples/snippets/generated" + +deep-preserve-regex: +- "/java-bare-metal-solution/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" + +deep-copy-regex: +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/proto-google-.*/src" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/proto-google-cloud-bare-metal-solution-$1/src" +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/grpc-google-.*/src" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/grpc-google-cloud-bare-metal-solution-$1/src" +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/gapic-google-.*/src" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/google-cloud-bare-metal-solution/src" +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/samples/snippets/generated" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/samples/snippets/generated" + +api-name: baremetalsolution \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json new file mode 100644 index 0000000000..2cccd4b889 --- /dev/null +++ b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json @@ -0,0 +1,16 @@ +{ + "api_shortname": "secretmanager", + "name_pretty": "Secret Management", + "product_documentation": "https://cloud.google.com/solutions/secrets-management/", + "api_description": "allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-secretmanager/latest/overview", + "release_level": "preview", + "transport": "http", + "language": "java", + "repo": "googleapis/google-cloud-java", + "repo_short": "java-secretmanager", + "distribution_name": "com.google.cloud:google-cloud-secretmanager", + "api_id": "secretmanager.googleapis.com", + "library_type": "GAPIC_AUTO", + "requires_billing": true +} \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json new file mode 100644 index 0000000000..0b3bbc2b55 --- /dev/null +++ b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json @@ -0,0 +1,20 @@ +{ + "api_shortname": "baremetalsolution", + "name_pretty": "Bare Metal Solution", + "product_documentation": "https://cloud.google.com/bare-metal/docs", + "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", + "release_level": "preview", + "transport": "grpc", + "language": "java", + "repo": "googleapis/google-cloud-java", + "repo_short": "java-bare-metal-solution", + "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", + "api_id": "baremetalsolution.googleapis.com", + "library_type": "GAPIC_AUTO", + "requires_billing": true, + "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", + "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", + "recommended_package": "com.google.example", + "min_java_version": 8 +} \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json new file mode 100644 index 0000000000..b39c297a85 --- /dev/null +++ b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json @@ -0,0 +1,21 @@ +{ + "api_shortname": "baremetalsolution", + "name_pretty": "Bare Metal Solution", + "product_documentation": "https://cloud.google.com/bare-metal/docs", + "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", + "release_level": "preview", + "transport": "grpc", + "language": "java", + "repo": "googleapis/java-bare-metal-solution", + "repo_short": "java-bare-metal-solution", + "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", + "api_id": "baremetalsolution.googleapis.com", + "library_type": "GAPIC_COMBO", + "requires_billing": true, + "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", + "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", + "extra_versioned_modules": "test-module", + "recommended_package": "com.google.example", + "min_java_version": 8 +} \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json new file mode 100644 index 0000000000..7730cd6987 --- /dev/null +++ b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json @@ -0,0 +1,19 @@ +{ + "api_shortname": "baremetalsolution", + "name_pretty": "Bare Metal Solution", + "product_documentation": "https://cloud.google.com/bare-metal/docs", + "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", + "release_level": "preview", + "transport": "grpc", + "language": "java", + "repo": "googleapis/sdk-platform-java", + "repo_short": "java-bare-metal-solution", + "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", + "library_type": "OTHER", + "requires_billing": true, + "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", + "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", + "recommended_package": "com.google.example", + "min_java_version": 8 +} \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py b/hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py new file mode 100644 index 0000000000..2ba11e6bba --- /dev/null +++ b/hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py @@ -0,0 +1,36 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import synthtool as s +from synthtool.languages import java + + +for library in s.get_staging_dirs(): + # put any special-case replacements here + s.move(library) + +s.remove_staging_dirs() +java.common_templates(monorepo=True, excludes=[ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "java.header", + "license-checks.xml", + "renovate.json", + ".gitignore" +]) \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/pr_description-golden.txt b/hermetic_build/release_note_generation/tests/resources/goldens/pr_description-golden.txt new file mode 100644 index 0000000000..1a0f874936 --- /dev/null +++ b/hermetic_build/release_note_generation/tests/resources/goldens/pr_description-golden.txt @@ -0,0 +1,17 @@ +This pull request is generated with proto changes between [googleapis/googleapis@3b6f144](https://github.com/googleapis/googleapis/commit/3b6f144d47b0a1d2115ab2445ec06e80cc324a44) (exclusive) and [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) (inclusive). + +BEGIN_COMMIT_OVERRIDE +BEGIN_NESTED_COMMIT +fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +chore: update the libraries_bom version to 2.3.4 +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +feat: Make Layout Parser generally available in V1 + +PiperOrigin-RevId: 638924855 + +Source Link: [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) +END_NESTED_COMMIT +END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt b/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt new file mode 100644 index 0000000000..81f7c0c4d2 --- /dev/null +++ b/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt @@ -0,0 +1,10 @@ +This pull request is generated with proto changes between [googleapis/googleapis@3b6f144](https://github.com/googleapis/googleapis/commit/3b6f144d47b0a1d2115ab2445ec06e80cc324a44) (exclusive) and [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) (inclusive). + +BEGIN_COMMIT_OVERRIDE +BEGIN_NESTED_COMMIT +fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 +END_NESTED_COMMIT +BEGIN_NESTED_COMMIT +chore: update the libraries_bom version to 2.3.4 +END_NESTED_COMMIT +END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt b/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt new file mode 100644 index 0000000000..f89901f8c1 --- /dev/null +++ b/hermetic_build/release_note_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt @@ -0,0 +1,5 @@ +BEGIN_COMMIT_OVERRIDE +BEGIN_NESTED_COMMIT +fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 +END_NESTED_COMMIT +END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml b/hermetic_build/release_note_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml new file mode 100644 index 0000000000..e69de29bb2 From deadfb5aac9f3e138fb16780e57c4573ea188e75 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 14:39:53 -0400 Subject: [PATCH 19/69] add init --- hermetic_build/common/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 hermetic_build/common/tests/__init__.py diff --git a/hermetic_build/common/tests/__init__.py b/hermetic_build/common/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From fcebe5ce56a8188cbb39ffeae25339e1842c9ea4 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 14:44:55 -0400 Subject: [PATCH 20/69] lint --- .../tests/owlbot/java_unit_tests.py | 1 - .../tests/utilities_unit_tests.py | 1 - .../generate_pr_description.py | 2 +- .../commit_message_formatter_unit_tests.py | 2 +- .../tests/resources/goldens/owlbot-golden.py | 29 ++++++++++--------- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/hermetic_build/library_generation/tests/owlbot/java_unit_tests.py b/hermetic_build/library_generation/tests/owlbot/java_unit_tests.py index e59486f6ad..9b39d0b544 100644 --- a/hermetic_build/library_generation/tests/owlbot/java_unit_tests.py +++ b/hermetic_build/library_generation/tests/owlbot/java_unit_tests.py @@ -66,7 +66,6 @@ class JavaUnitTests(unittest.TestCase): - def test_version_from_maven_metadata(self): self.assertEqual("3.3.0", java.version_from_maven_metadata(SAMPLE_METADATA)) diff --git a/hermetic_build/library_generation/tests/utilities_unit_tests.py b/hermetic_build/library_generation/tests/utilities_unit_tests.py index b8bb9c6149..eae0e91fcf 100644 --- a/hermetic_build/library_generation/tests/utilities_unit_tests.py +++ b/hermetic_build/library_generation/tests/utilities_unit_tests.py @@ -257,7 +257,6 @@ def test_generate_postprocessing_prerequisite_files__custom_transport_set_in_con def test_create__library_invalid_transport__fails( self, ): - with self.assertRaises(ValueError): test_library_with_invalid_transport = LibraryConfig( api_shortname="secretmanager", diff --git a/hermetic_build/release_note_generation/generate_pr_description.py b/hermetic_build/release_note_generation/generate_pr_description.py index 64119a1283..b71facd7a4 100755 --- a/hermetic_build/release_note_generation/generate_pr_description.py +++ b/hermetic_build/release_note_generation/generate_pr_description.py @@ -24,7 +24,7 @@ format_commit_message, format_repo_level_change, commit_link, - wrap_override_commit + wrap_override_commit, ) EMPTY_MESSAGE = "" diff --git a/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py b/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py index 8b4fccf2a7..ac888cea7b 100644 --- a/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py +++ b/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py @@ -24,7 +24,7 @@ commit_link, format_repo_level_change, wrap_googleapis_commit, - wrap_override_commit + wrap_override_commit, ) gen_config = GenerationConfig( diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py b/hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py index 2ba11e6bba..7559eaf034 100644 --- a/hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py +++ b/hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py @@ -21,16 +21,19 @@ s.move(library) s.remove_staging_dirs() -java.common_templates(monorepo=True, excludes=[ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "SECURITY.md", - "java.header", - "license-checks.xml", - "renovate.json", - ".gitignore" -]) \ No newline at end of file +java.common_templates( + monorepo=True, + excludes=[ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "java.header", + "license-checks.xml", + "renovate.json", + ".gitignore", + ], +) From 0b400e9b97b1aeefdf45efac9c108dfbe2564cae Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 14:45:09 -0400 Subject: [PATCH 21/69] change ci --- .../workflows/verify_library_generation.yaml | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index 722ea7db95..99da0ece8a 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -80,23 +80,24 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 - name: install python dependencies shell: bash run: | set -ex - pushd library_generation - pip install --require-hashes -r requirements.txt - pip install . + pushd hermetic_build + pip install common + pip install library_generation + pip install release_note_generation popd - name: Run shell unit tests run: | set -x - library_generation/test/generate_library_unit_tests.sh + hermetic_build/library_generation/tests/generate_library_unit_tests.sh - name: Run python unit tests run: | set -x - python -m unittest discover -s library_generation/test/ -p "*unit_tests.py" + python -m unittest discover -s hermetic_build -p "*unit_tests.py" library-generation-lint-shell: runs-on: ubuntu-22.04 needs: should-run-library-generation-tests @@ -106,7 +107,7 @@ jobs: - name: Run ShellCheck uses: ludeeus/action-shellcheck@2.0.0 with: - scandir: 'library_generation' + scandir: 'hermetic_build' format: tty severity: error library-generation-lint-python: @@ -119,12 +120,14 @@ jobs: shell: bash run: | set -ex - pushd library_generation - pip install --require-hashes -r requirements.txt + pushd hermetic_build + pip install common + pip install library_generation + pip install release_note_generation popd - name: Lint shell: bash run: | # exclude generated golden files # exclude owlbot until further refaction - black --check library_generation --exclude "(library_generation/test/resources/goldens)" + black --check hermetic_build --exclude "(library_generation/tests/resources/goldens)" From b32341206361c509a180ddbb43dc4ee5e8d0d741 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 14:53:34 -0400 Subject: [PATCH 22/69] change integration in ci --- .github/workflows/verify_library_generation.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index 99da0ece8a..d98336c326 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 - name: install pyenv shell: bash run: | @@ -63,15 +63,16 @@ jobs: shell: bash run: | set -ex - pushd library_generation - pip install --require-hashes -r requirements.txt - pip install . + pushd hermetic_build + pip install common + pip install library_generation + pip install release_note_generation popd - name: Run integration tests shell: bash run: | set -x - python -m unittest library_generation/test/integration_tests.py + python -m unittest hermetic_build/library_generation/tests/integration_tests.py library-generation-unit-tests: runs-on: ubuntu-22.04 needs: should-run-library-generation-tests From 104de86b84a2ce9a8c33f4dabb106ca2a269a925 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 14:53:46 -0400 Subject: [PATCH 23/69] change dockerfile --- .../library_generation/library_generation.Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 3131028252..7a605deedf 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -45,7 +45,8 @@ RUN apt-get update && apt-get install -y \ && apt-get clean # copy source code -COPY library_generation /src +COPY hermetic_build/library_generation /src/library_generation +COPY hermetic_build/common /src/common # install protoc WORKDIR /protoc @@ -79,8 +80,8 @@ RUN python -m pip install --upgrade pip # install main scripts as a python package WORKDIR /src -RUN python -m pip install --require-hashes -r requirements.txt -RUN python -m pip install . +RUN python -m pip install common +RUN python -m pip install library_generation # Install nvm with node and npm ENV NODE_VERSION 20.12.0 From 1dff8b5141caedc281a6dbed7ef509a0ded2cfee Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:15:11 -0400 Subject: [PATCH 24/69] setup venv --- .github/workflows/verify_library_generation.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index d98336c326..3b6e7b56a6 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -86,6 +86,8 @@ jobs: shell: bash run: | set -ex + python -m venv .venv + source .venv/bin/activate pushd hermetic_build pip install common pip install library_generation @@ -117,10 +119,15 @@ jobs: if: needs.should-run-library-generation-tests.outputs.should_run == 'true' steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.12 - name: install python dependencies shell: bash run: | set -ex + python -m venv .venv + source .venv/bin/activate pushd hermetic_build pip install common pip install library_generation From 78d22e12f1c895248eb837507addfa7a04820c41 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:16:08 -0400 Subject: [PATCH 25/69] change dir --- .github/workflows/verify_library_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index 3b6e7b56a6..c52b890921 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -27,7 +27,7 @@ jobs: git checkout -b "${head_ref}" fork/${head_ref} changed_directories="$(git diff --name-only "fork/${head_ref}" "origin/${base_ref}")" fi - if [[ ${changed_directories} =~ "library_generation/" ]]; then + if [[ ${changed_directories} =~ "hermetic_build/" ]]; then echo "should_run=true" >> $GITHUB_OUTPUT else echo "should_run=false" >> $GITHUB_OUTPUT From d4d01b1fd11f093af82e2a1c15c1d4ec1f184510 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:18:55 -0400 Subject: [PATCH 26/69] change dir --- .github/workflows/verify_library_generation.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index c52b890921..164dfa058d 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -88,11 +88,9 @@ jobs: set -ex python -m venv .venv source .venv/bin/activate - pushd hermetic_build - pip install common - pip install library_generation - pip install release_note_generation - popd + pip install hermetic_build/common + pip install hermetic_build/library_generation + pip install hermetic_build/release_note_generation - name: Run shell unit tests run: | set -x From 82938676a9b655f0d0f88d388d3fa9ae18bcc340 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:22:35 -0400 Subject: [PATCH 27/69] remove venv --- .github/workflows/verify_library_generation.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index 164dfa058d..afaccddb89 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -86,8 +86,7 @@ jobs: shell: bash run: | set -ex - python -m venv .venv - source .venv/bin/activate + pip install --upgrade pip pip install hermetic_build/common pip install hermetic_build/library_generation pip install hermetic_build/release_note_generation @@ -124,13 +123,10 @@ jobs: shell: bash run: | set -ex - python -m venv .venv - source .venv/bin/activate - pushd hermetic_build - pip install common - pip install library_generation - pip install release_note_generation - popd + pip install --upgrade pip + pip install hermetic_build/common + pip install hermetic_build/library_generation + pip install hermetic_build/release_note_generation - name: Lint shell: bash run: | From 6a12b865a7cdd0a6296b3be1aaa06469a179770e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:24:48 -0400 Subject: [PATCH 28/69] add black --- hermetic_build/common/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hermetic_build/common/setup.py b/hermetic_build/common/setup.py index 5e2dc0f55a..c3c69ce59f 100755 --- a/hermetic_build/common/setup.py +++ b/hermetic_build/common/setup.py @@ -12,6 +12,7 @@ "common": ".", }, install_requires=[ + "black==24.8.0", "parameterized==0.9.0", "PyYAML==6.0.2", ], From c58eb24fa714879cfca97849edcc79d300cbd35a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:30:29 -0400 Subject: [PATCH 29/69] change it in ci --- .../workflows/verify_library_generation.yaml | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index afaccddb89..8c12704b11 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -47,27 +47,13 @@ jobs: - uses: actions/setup-python@v5 with: python-version: 3.12 - - name: install pyenv - shell: bash - run: | - set -ex - curl https://pyenv.run | bash - # setup environment - export PYENV_ROOT="$HOME/.pyenv" - export PATH="$PYENV_ROOT/bin:$PATH" - echo "PYENV_ROOT=${PYENV_ROOT}" >> $GITHUB_ENV - echo "PATH=${PATH}" >> $GITHUB_ENV - - set +ex - name: install python dependencies shell: bash run: | set -ex - pushd hermetic_build - pip install common - pip install library_generation - pip install release_note_generation - popd + pip install --upgrade pip + pip install hermetic_build/common + pip install hermetic_build/library_generation - name: Run integration tests shell: bash run: | From 80f8a292f711e934a7e241be6dac3ae458ea4f6a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:31:50 -0400 Subject: [PATCH 30/69] remove release note gen in integration test --- .../tests/integration_tests.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index b189c9481b..04b70cdcf1 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from click.testing import CliRunner import difflib import json import tempfile @@ -25,9 +24,6 @@ from distutils.dir_util import copy_tree from distutils.file_util import copy_file from pathlib import Path -from library_generation.cli.generate_release_note import ( - generate as generate_pr_description, -) from common.model.generation_config import GenerationConfig from common.model.generation_config import from_yaml from library_generation.tests.compare_poms import compare_xml @@ -99,18 +95,7 @@ def test_entry_point_running_in_container(self): current_config=current_config_name, api_definition=api_definitions_path, ) - # 4. generate pr description - # noinspection PyTypeChecker - result = CliRunner().invoke( - generate_pr_description, - [ - f"--baseline-generation-config-path={config_location}/{baseline_config_name}", - f"--current-generation-config-path={config_location}/{current_config_name}", - f"--repository-path={repo_location}", - ], - ) - self.assertEqual(0, result.exit_code) - # 5. compare generation result with golden files + # 4. compare generation result with golden files print( "Generation finished successfully. " "Will now compare differences between generated and existing " From cec724c4161ee132d61be2b7e8747c655f392709 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:36:52 -0400 Subject: [PATCH 31/69] remove deprecated pkg in it --- .../library_generation/tests/integration_tests.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index 04b70cdcf1..ec556603be 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -21,8 +21,6 @@ import shutil import subprocess import unittest -from distutils.dir_util import copy_tree -from distutils.file_util import copy_file from pathlib import Path from common.model.generation_config import GenerationConfig from common.model.generation_config import from_yaml @@ -283,14 +281,14 @@ def __prepare_golden_files( ): for library_name in library_names: if config.is_monorepo(): - copy_tree(f"{repo_dest}/{library_name}", f"{golden_dir}/{library_name}") - copy_tree( + shutil.copytree(f"{repo_dest}/{library_name}", f"{golden_dir}/{library_name}") + shutil.copytree( f"{repo_dest}/gapic-libraries-bom", f"{golden_dir}/gapic-libraries-bom", ) - copy_file(f"{repo_dest}/pom.xml", golden_dir) + shutil.copyfile(f"{repo_dest}/pom.xml", golden_dir) else: - copy_tree(f"{repo_dest}", f"{golden_dir}/{library_name}") + shutil.copytree(f"{repo_dest}", f"{golden_dir}/{library_name}") @classmethod def __run_entry_point_in_docker_container( From 8c1e5b222e7f351f7831f763d86c3401a76e41bf Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:42:09 -0400 Subject: [PATCH 32/69] change dir in it --- hermetic_build/library_generation/tests/integration_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index ec556603be..7325404990 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -32,7 +32,7 @@ config_dir = os.path.join(script_dir, "resources", "integration") golden_dir = os.path.join(config_dir, "golden") generator_jar_coordinates_file = os.path.join(config_dir, "test_generator_coordinates") -repo_root_dir = os.path.join(script_dir, "..", "..") +repo_root_dir = os.path.join(script_dir, "..", "..", "..") build_file = os.path.join( repo_root_dir, ".cloudbuild", "library_generation", "library_generation.Dockerfile" ) From 2afcd90773d8b46f680bba56b735347ec1650d45 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:44:03 -0400 Subject: [PATCH 33/69] lint --- hermetic_build/library_generation/tests/integration_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index 7325404990..ad7bc7a9bb 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -281,7 +281,9 @@ def __prepare_golden_files( ): for library_name in library_names: if config.is_monorepo(): - shutil.copytree(f"{repo_dest}/{library_name}", f"{golden_dir}/{library_name}") + shutil.copytree( + f"{repo_dest}/{library_name}", f"{golden_dir}/{library_name}" + ) shutil.copytree( f"{repo_dest}/gapic-libraries-bom", f"{golden_dir}/gapic-libraries-bom", From bfb69d7bee342328956ec8310eaa5b57b0b49813 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:46:54 -0400 Subject: [PATCH 34/69] fix dir in dockerfile --- .../library_generation/library_generation.Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 7a605deedf..432e267210 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -50,7 +50,7 @@ COPY hermetic_build/common /src/common # install protoc WORKDIR /protoc -RUN source /src/utils/utilities.sh \ +RUN source /src/library_generation/utils/utilities.sh \ && download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}" # we indicate protoc is available in the container via env vars ENV DOCKER_PROTOC_LOCATION=/protoc @@ -58,7 +58,7 @@ ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}" # install grpc WORKDIR /grpc -RUN source /src/utils/utilities.sh \ +RUN source /src/library_generation/utils/utilities.sh \ && download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}" # similar to protoc, we indicate grpc is available in the container via env vars ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe" @@ -121,4 +121,4 @@ RUN chmod -R a+rw /home RUN chmod -R a+rx /home/.nvm WORKDIR /workspace -ENTRYPOINT [ "python", "/src/cli/entry_point.py", "generate" ] +ENTRYPOINT [ "python", "/src/library_generation/cli/entry_point.py", "generate" ] From 6834fbcf5878c40a782bc6540db9b375ff29e815 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 15:57:16 -0400 Subject: [PATCH 35/69] change dir --- .../library_generation.Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 432e267210..b404913823 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -45,8 +45,8 @@ RUN apt-get update && apt-get install -y \ && apt-get clean # copy source code -COPY hermetic_build/library_generation /src/library_generation COPY hermetic_build/common /src/common +COPY hermetic_build/library_generation /src/library_generation # install protoc WORKDIR /protoc @@ -72,16 +72,16 @@ ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}" COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar" RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar" -# use python 3.11 (the base image has several python versions; here we define the default one) +# use python 3.12 (the base image has several python versions; here we define the default one) RUN rm $(which python3) -RUN ln -s $(which python3.11) /usr/local/bin/python -RUN ln -s $(which python3.11) /usr/local/bin/python3 +RUN ln -s $(which python3.12) /usr/local/bin/python +RUN ln -s $(which python3.12) /usr/local/bin/python3 RUN python -m pip install --upgrade pip # install main scripts as a python package -WORKDIR /src -RUN python -m pip install common -RUN python -m pip install library_generation +WORKDIR / +RUN python -m pip install src/common +RUN python -m pip install src/library_generation # Install nvm with node and npm ENV NODE_VERSION 20.12.0 From 7042b87d0f07471bf281aad6749eeadbdbf6e58c Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 16:10:18 -0400 Subject: [PATCH 36/69] fix copyfile --- hermetic_build/library_generation/tests/integration_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index 10eb4056d2..85f56c9c96 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -288,7 +288,7 @@ def __prepare_golden_files( f"{repo_dest}/gapic-libraries-bom", f"{golden_dir}/gapic-libraries-bom", ) - shutil.copyfile(f"{repo_dest}/pom.xml", golden_dir) + shutil.copyfile(f"{repo_dest}/pom.xml", f"{golden_dir}/pom.xml") else: shutil.copytree(f"{repo_dest}", f"{golden_dir}/{library_name}") From 06ebe0bcf97f84010022cc14ea1e65980ff8bdbe Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 16:26:53 -0400 Subject: [PATCH 37/69] fix copytree --- hermetic_build/library_generation/tests/integration_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index 85f56c9c96..8612821684 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -287,6 +287,7 @@ def __prepare_golden_files( shutil.copytree( f"{repo_dest}/gapic-libraries-bom", f"{golden_dir}/gapic-libraries-bom", + dirs_exist_ok=True, ) shutil.copyfile(f"{repo_dest}/pom.xml", f"{golden_dir}/pom.xml") else: From f576c1ac87f12de65c13702b66f17ac16684fbcb Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 16:30:24 -0400 Subject: [PATCH 38/69] fix path in script --- .github/scripts/hermetic_library_generation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 1fb4b7973a..a71f393ee7 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -115,7 +115,7 @@ docker run \ -e GENERATOR_VERSION="${image_tag}" \ --entrypoint python \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ - /src/cli/generate_release_note.py generate \ + /src/release_note_generation/cli/generate_release_note.py generate \ --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ --current-generation-config-path="${workspace_name}/${generation_config}" \ --repository-path="${workspace_name}" From de6b8703221c2a000dc771f474533340d45d823f Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 16:37:33 -0400 Subject: [PATCH 39/69] remove pr description comp in it --- .../library_generation/tests/integration_tests.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index 8612821684..23fcfdc4f5 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -179,16 +179,6 @@ def test_entry_point_running_in_container(self): ) ) print(" pom.xml comparison succeed.") - # compare PR description - description_file = f"{output_dir}/{repo}/pr_description.txt" - self.assertTrue( - cmp( - f"{config_dir}/{repo}/pr-description-golden.txt", - f"{description_file}", - ), - "The generated PR description does not match the expected golden file", - ) - print(" PR description comparison succeed.") self.__remove_generated_files() shutil.rmtree(api_definitions_path) From 3c03477857cd932632baf81663ad316de55f9b2e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 16:38:57 -0400 Subject: [PATCH 40/69] lint --- hermetic_build/library_generation/tests/integration_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index 23fcfdc4f5..fd534ff207 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -14,7 +14,6 @@ import difflib import json import tempfile -from filecmp import cmp from filecmp import dircmp from git import Repo import os From 3b132ce3dfaeaab06f8766fc9c1e05c09a1d657e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 17:05:53 -0400 Subject: [PATCH 41/69] fix showcase --- showcase/scripts/generate_showcase.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/showcase/scripts/generate_showcase.sh b/showcase/scripts/generate_showcase.sh index 82691e371d..36e72ad76a 100755 --- a/showcase/scripts/generate_showcase.sh +++ b/showcase/scripts/generate_showcase.sh @@ -6,8 +6,8 @@ set -ex readonly SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -readonly LIB_GEN_SCRIPTS_DIR="${SCRIPT_DIR}/../../library_generation/" -source "${LIB_GEN_SCRIPTS_DIR}/test/test_utilities.sh" +readonly LIB_GEN_SCRIPTS_DIR="${SCRIPT_DIR}/../../hermetic_build/library_generation/" +source "${LIB_GEN_SCRIPTS_DIR}/tests/test_utilities.sh" source "${LIB_GEN_SCRIPTS_DIR}/utils/utilities.sh" readonly perform_cleanup=$1 From b9c5374376a222b72dbb39baca6d4e63d0c83e60 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 18 Oct 2024 17:09:30 -0400 Subject: [PATCH 42/69] fix showcase --- showcase/scripts/generate_showcase.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/showcase/scripts/generate_showcase.sh b/showcase/scripts/generate_showcase.sh index 36e72ad76a..cfb3c713ae 100755 --- a/showcase/scripts/generate_showcase.sh +++ b/showcase/scripts/generate_showcase.sh @@ -106,7 +106,7 @@ include_samples="false" rm -rdf output/showcase-output mkdir output/showcase-output set +e -bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \ +bash "${SCRIPT_DIR}/../../hermetic_build/library_generation/generate_library.sh" \ --protoc_version "${protoc_version}" \ --grpc_version "${grpc_version}" \ --proto_path "schema/google/showcase/v1beta1" \ From d35b9aa8231f66b1326e910b3b4c6668e97196f3 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 08:36:53 -0400 Subject: [PATCH 43/69] setup python in composite action --- .github/scripts/action.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index cf6a9f6360..1961cd76f7 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -37,6 +37,9 @@ inputs: runs: using: "composite" steps: + - uses: actions/setup-python@v5 + with: + python-version: 3.12 - name: Copy shell script shell: bash run: | From 123b406c6e7115a2d397f40b03b6d108122a73da Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 08:47:38 -0400 Subject: [PATCH 44/69] install deps --- .github/scripts/action.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 1961cd76f7..0c9672abb6 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -40,6 +40,12 @@ runs: - uses: actions/setup-python@v5 with: python-version: 3.12 + - name: Install dependency + shell: bash + run: | + cd ${{ github.action_path }}/../../hermetic_build + pip install common + pip install release_note_generation - name: Copy shell script shell: bash run: | From 04f410125c176d8be471aada23ada5fa2cff5ea6 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 08:56:09 -0400 Subject: [PATCH 45/69] change path --- .github/scripts/action.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 0c9672abb6..06d7d66f34 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -43,9 +43,10 @@ runs: - name: Install dependency shell: bash run: | - cd ${{ github.action_path }}/../../hermetic_build - pip install common - pip install release_note_generation + # repository root + cd ${{ github.action_path }}/../../ + pip install hermetic_build/common + pip install hermetic_build/release_note_generation - name: Copy shell script shell: bash run: | From 54d5116a77b4ed845eb9feb9a996ce208e94609d Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 09:33:55 -0400 Subject: [PATCH 46/69] copy and install --- .github/scripts/action.yaml | 17 +++++++++++++---- .github/scripts/hermetic_library_generation.sh | 18 ++++-------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 06d7d66f34..04534ce73b 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -40,18 +40,27 @@ runs: - uses: actions/setup-python@v5 with: python-version: 3.12 - - name: Install dependency + - name: Copy python script shell: bash run: | # repository root cd ${{ github.action_path }}/../../ - pip install hermetic_build/common - pip install hermetic_build/release_note_generation + rsync -av \ + --exclude=hermetic_build/library_generation \ + --exclude=hermetic_build/common/tests \ + --exclude=hermetic_build/release_note_generation/tests \ + hermetic_build/ "${GITHUB_WORKSPACE}" - name: Copy shell script shell: bash run: | cd ${{ github.action_path }} - cp hermetic_library_generation.sh $GITHUB_WORKSPACE + cp hermetic_library_generation.sh "${GITHUB_WORKSPACE}" + - name: Install python package + shell: bash + run: | + cd "${GITHUB_WORKSPACE}" + pip install hermetic_build/common + pip install hermetic_build/release_note_generation - name: Generate changed libraries shell: bash run: | diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index a71f393ee7..b1c97f11d1 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -106,26 +106,16 @@ docker run \ --current-generation-config-path="${workspace_name}/${generation_config}" \ --api-definitions-path="${workspace_name}/googleapis" -# generate pr description -docker run \ - --rm \ - --quiet \ - -u "$(id -u):$(id -g)" \ - -v "$(pwd):${workspace_name}" \ - -e GENERATOR_VERSION="${image_tag}" \ - --entrypoint python \ - gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ - /src/release_note_generation/cli/generate_release_note.py generate \ - --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ - --current-generation-config-path="${workspace_name}/${generation_config}" \ - --repository-path="${workspace_name}" +python hermetic_build/release_note_generation/cli/generate_release_note.py generation \ + --baseline-generation-config-path="${baseline_generation_config}" \ + --current-generation-config-path="${generation_config}" # remove api definitions after generation rm -rf "${api_def_dir}" # commit the change to the pull request. rm -rdf output googleapis "${baseline_generation_config}" -git add --all -- ':!pr_description.txt' ':!hermetic_library_generation.sh' +git add --all -- ':!pr_description.txt' ':!hermetic_library_generation.sh' ':!hermetic_build' changed_files=$(git diff --cached --name-only) if [[ "${changed_files}" != "" ]]; then echo "Commit changes..." From 26436a762d57ae39529bc57f933fc311f05eedf9 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 09:50:01 -0400 Subject: [PATCH 47/69] reduce log --- .github/scripts/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 04534ce73b..2280b3e03f 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -45,7 +45,7 @@ runs: run: | # repository root cd ${{ github.action_path }}/../../ - rsync -av \ + rsync -a \ --exclude=hermetic_build/library_generation \ --exclude=hermetic_build/common/tests \ --exclude=hermetic_build/release_note_generation/tests \ From 5a5465f4071938a5c45c7c89edc5e07212db471b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 09:54:54 -0400 Subject: [PATCH 48/69] debug --- .github/scripts/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 2280b3e03f..5beceaab8f 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -59,6 +59,7 @@ runs: shell: bash run: | cd "${GITHUB_WORKSPACE}" + ls -l hermetic_build pip install hermetic_build/common pip install hermetic_build/release_note_generation - name: Generate changed libraries From 8458e3b2c8c3f6359e15d009f9b81606de7ef132 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 09:57:09 -0400 Subject: [PATCH 49/69] debug --- .github/scripts/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 5beceaab8f..31daa0763f 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -45,6 +45,7 @@ runs: run: | # repository root cd ${{ github.action_path }}/../../ + ls -l hermetic_build rsync -a \ --exclude=hermetic_build/library_generation \ --exclude=hermetic_build/common/tests \ From 4d321dc9162a3d56d862c1c6c712ac368250f927 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 10:04:41 -0400 Subject: [PATCH 50/69] debug --- .github/scripts/action.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 31daa0763f..6154b988fb 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -45,18 +45,17 @@ runs: run: | # repository root cd ${{ github.action_path }}/../../ - ls -l hermetic_build rsync -a \ - --exclude=hermetic_build/library_generation \ - --exclude=hermetic_build/common/tests \ - --exclude=hermetic_build/release_note_generation/tests \ - hermetic_build/ "${GITHUB_WORKSPACE}" + --exclude=hermetic_build/library_generation/ \ + --exclude=hermetic_build/common/tests/ \ + --exclude=hermetic_build/release_note_generation/tests/ \ + hermetic_build/ "${GITHUB_WORKSPACE}/" - name: Copy shell script shell: bash run: | cd ${{ github.action_path }} cp hermetic_library_generation.sh "${GITHUB_WORKSPACE}" - - name: Install python package + - name: Install python packages shell: bash run: | cd "${GITHUB_WORKSPACE}" From 8f3865e37c361db67cdaeed887219659ca1b5282 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 10:12:18 -0400 Subject: [PATCH 51/69] debug --- .github/scripts/action.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 6154b988fb..54664ba91a 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -43,6 +43,7 @@ runs: - name: Copy python script shell: bash run: | + set -x # repository root cd ${{ github.action_path }}/../../ rsync -a \ @@ -50,6 +51,7 @@ runs: --exclude=hermetic_build/common/tests/ \ --exclude=hermetic_build/release_note_generation/tests/ \ hermetic_build/ "${GITHUB_WORKSPACE}/" + ls -l "${GITHUB_WORKSPACE}/hermetic_build" - name: Copy shell script shell: bash run: | From 49f6cf3a39300319ba05b0623eaad423d169be57 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 10:29:49 -0400 Subject: [PATCH 52/69] debug --- .github/scripts/action.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 54664ba91a..4c6bc81147 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -46,11 +46,10 @@ runs: set -x # repository root cd ${{ github.action_path }}/../../ - rsync -a \ - --exclude=hermetic_build/library_generation/ \ - --exclude=hermetic_build/common/tests/ \ - --exclude=hermetic_build/release_note_generation/tests/ \ - hermetic_build/ "${GITHUB_WORKSPACE}/" + rsync -vR \ + --exclude=library_generation \ + --exclude=tests \ + hermetic_build "${GITHUB_WORKSPACE}" ls -l "${GITHUB_WORKSPACE}/hermetic_build" - name: Copy shell script shell: bash From aceba8462d56c5e47a806f6644f9c19d50560365 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 10:32:56 -0400 Subject: [PATCH 53/69] debug --- .github/scripts/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 4c6bc81147..875b7a3bf8 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -46,7 +46,7 @@ runs: set -x # repository root cd ${{ github.action_path }}/../../ - rsync -vR \ + rsync -rvR \ --exclude=library_generation \ --exclude=tests \ hermetic_build "${GITHUB_WORKSPACE}" From 4b98eecc9d83d7f5c5701c727505272bfda41f7f Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 10:33:04 -0400 Subject: [PATCH 54/69] debug --- .github/scripts/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 875b7a3bf8..804483a12c 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -46,7 +46,7 @@ runs: set -x # repository root cd ${{ github.action_path }}/../../ - rsync -rvR \ + rsync -rv \ --exclude=library_generation \ --exclude=tests \ hermetic_build "${GITHUB_WORKSPACE}" From 71bd36133c14a50215913698fe7c392c84e42ea2 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 10:37:50 -0400 Subject: [PATCH 55/69] remove debug --- .github/scripts/action.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 804483a12c..039bfb6ec0 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -50,7 +50,6 @@ runs: --exclude=library_generation \ --exclude=tests \ hermetic_build "${GITHUB_WORKSPACE}" - ls -l "${GITHUB_WORKSPACE}/hermetic_build" - name: Copy shell script shell: bash run: | @@ -60,7 +59,6 @@ runs: shell: bash run: | cd "${GITHUB_WORKSPACE}" - ls -l hermetic_build pip install hermetic_build/common pip install hermetic_build/release_note_generation - name: Generate changed libraries From ec17d849374b672f065b965b77c96eebe4047a5f Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 10:45:07 -0400 Subject: [PATCH 56/69] install library_generation --- .github/scripts/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 039bfb6ec0..447c17e25a 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -47,7 +47,6 @@ runs: # repository root cd ${{ github.action_path }}/../../ rsync -rv \ - --exclude=library_generation \ --exclude=tests \ hermetic_build "${GITHUB_WORKSPACE}" - name: Copy shell script @@ -60,6 +59,7 @@ runs: run: | cd "${GITHUB_WORKSPACE}" pip install hermetic_build/common + pip install hermetic_build/library_generation pip install hermetic_build/release_note_generation - name: Generate changed libraries shell: bash From 180803adfd8a9b575c036a887228ab8de25f30ac Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 10:50:21 -0400 Subject: [PATCH 57/69] change cmd --- .github/scripts/hermetic_library_generation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index b1c97f11d1..59fc82fc12 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -106,7 +106,7 @@ docker run \ --current-generation-config-path="${workspace_name}/${generation_config}" \ --api-definitions-path="${workspace_name}/googleapis" -python hermetic_build/release_note_generation/cli/generate_release_note.py generation \ +python hermetic_build/release_note_generation/cli/generate_release_note.py generate \ --baseline-generation-config-path="${baseline_generation_config}" \ --current-generation-config-path="${generation_config}" From e458552f6016ebbc01f97363679862951548eb0b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 11:42:22 -0400 Subject: [PATCH 58/69] remove unused test resources --- .../config_without_temp_excludes.yaml | 10 ---- .../generation_config_library_modified.yaml | 14 ----- ...on_config_with_duplicate_library_name.yaml | 51 ------------------- .../test-config/monorepo_baseline.yaml | 30 ----------- .../test-config/monorepo_current.yaml | 30 ----------- .../monorepo_with_common_protos.yaml | 43 ---------------- .../monorepo_without_common_protos.yaml | 33 ------------ .../goldens/pr_description-golden.txt | 17 ------- ...qualified_commit_pr_description-golden.txt | 10 ---- .../repo_level_only_pr_description-golden.txt | 5 -- .../misc/BUILD_comment_common_resources.bazel | 5 -- .../misc/BUILD_comment_iam_policy.bazel | 5 -- .../misc/BUILD_comment_locations.bazel | 5 -- .../misc/BUILD_common_resources.bazel | 5 -- .../resources/misc/BUILD_gapic_yaml.bazel | 3 -- .../tests/resources/misc/BUILD_grpc.bazel | 5 -- .../resources/misc/BUILD_grpc_rest.bazel | 5 -- .../resources/misc/BUILD_iam_locations.bazel | 6 --- .../resources/misc/BUILD_iam_policy.bazel | 5 -- .../misc/BUILD_include_samples_empty.bazel | 5 -- .../misc/BUILD_include_samples_false.bazel | 5 -- .../misc/BUILD_include_samples_true.bazel | 5 -- .../resources/misc/BUILD_locations.bazel | 5 -- .../misc/BUILD_no_additional_protos.bazel | 4 -- .../resources/misc/BUILD_no_gapic_yaml.bazel | 3 -- .../misc/BUILD_no_service_config.bazel | 3 -- .../misc/BUILD_no_service_yaml.bazel | 3 -- .../resources/misc/BUILD_proto_only.bazel | 16 ------ .../tests/resources/misc/BUILD_rest.bazel | 5 -- .../misc/BUILD_rest_numeric_enums_empty.bazel | 5 -- .../misc/BUILD_rest_numeric_enums_false.bazel | 5 -- .../misc/BUILD_rest_numeric_enums_true.bazel | 5 -- .../resources/misc/BUILD_service_config.bazel | 3 -- ...BUILD_service_config_relative_target.bazel | 3 -- .../resources/misc/BUILD_service_yaml.bazel | 3 -- .../BUILD_service_yaml_absolute_target.bazel | 3 -- .../config_without_api_description.yaml | 5 -- .../config_without_api_shortname.yaml | 4 -- .../config_without_gapics_key.yaml | 3 -- .../config_without_gapics_value.yaml | 4 -- .../config_without_googleapis.yaml | 8 --- .../test-config/config_without_libraries.yaml | 1 - .../config_without_library_value.yaml | 2 - .../config_without_name_pretty.yaml | 6 --- .../config_without_product_docs.yaml | 7 --- .../config_without_proto_path.yaml | 4 -- .../config_without_temp_excludes.yaml | 10 ---- .../test-monorepo/.github/.OwlBot.lock.yaml | 17 ------- .../test-service/.repo-metadata.json | 18 ------- .../empty_gen_config.yaml | 0 .../goldens/.OwlBot-hermetic-golden.yaml | 35 ------------- ...repo-metadata-custom-transport-golden.json | 16 ------ .../.repo-metadata-monorepo-golden.json | 20 -------- .../.repo-metadata-non-monorepo-golden.json | 21 -------- .../.repo-metadata-proto-only-golden.json | 19 ------- .../tests/resources/goldens/owlbot-golden.py | 39 -------------- 56 files changed, 607 deletions(-) delete mode 100644 hermetic_build/common/tests/resources/test-config/config_without_temp_excludes.yaml delete mode 100644 hermetic_build/common/tests/resources/test-config/generation_config_library_modified.yaml delete mode 100644 hermetic_build/common/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml delete mode 100644 hermetic_build/common/tests/resources/test-config/monorepo_baseline.yaml delete mode 100644 hermetic_build/common/tests/resources/test-config/monorepo_current.yaml delete mode 100644 hermetic_build/common/tests/resources/test-config/monorepo_with_common_protos.yaml delete mode 100644 hermetic_build/common/tests/resources/test-config/monorepo_without_common_protos.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/goldens/pr_description-golden.txt delete mode 100644 hermetic_build/library_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt delete mode 100644 hermetic_build/library_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_comment_common_resources.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_comment_iam_policy.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_comment_locations.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_common_resources.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_gapic_yaml.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_grpc.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_grpc_rest.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_iam_locations.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_iam_policy.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_empty.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_false.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_true.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_locations.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_no_additional_protos.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_no_gapic_yaml.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_config.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_yaml.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_proto_only.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_rest.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_service_config.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_service_config_relative_target.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_api_description.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_api_shortname.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_key.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_value.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_googleapis.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_libraries.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_library_value.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_name_pretty.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_product_docs.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_proto_path.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-config/config_without_temp_excludes.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-monorepo/.github/.OwlBot.lock.yaml delete mode 100644 hermetic_build/library_generation/tests/resources/test-monorepo/test-service/.repo-metadata.json delete mode 100644 hermetic_build/library_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml delete mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml delete mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json delete mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json delete mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json delete mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json delete mode 100644 hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py diff --git a/hermetic_build/common/tests/resources/test-config/config_without_temp_excludes.yaml b/hermetic_build/common/tests/resources/test-config/config_without_temp_excludes.yaml deleted file mode 100644 index 0d1bb7deea..0000000000 --- a/hermetic_build/common/tests/resources/test-config/config_without_temp_excludes.yaml +++ /dev/null @@ -1,10 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/common/tests/resources/test-config/generation_config_library_modified.yaml b/hermetic_build/common/tests/resources/test-config/generation_config_library_modified.yaml deleted file mode 100644 index f9ae96693b..0000000000 --- a/hermetic_build/common/tests/resources/test-config/generation_config_library_modified.yaml +++ /dev/null @@ -1,14 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 diff --git a/hermetic_build/common/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml b/hermetic_build/common/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml deleted file mode 100644 index c5613f4308..0000000000 --- a/hermetic_build/common/tests/resources/test-config/generation_config_with_duplicate_library_name.yaml +++ /dev/null @@ -1,51 +0,0 @@ -gapic_generator_version: 2.34.0 -protoc_version: 25.2 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 -template_excludes: - - ".github/*" - - ".kokoro/*" - - "samples/*" - - "CODE_OF_CONDUCT.md" - - "CONTRIBUTING.md" - - "LICENSE" - - "SECURITY.md" - - "java.header" - - "license-checks.xml" - - "renovate.json" - - ".gitignore" -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - codeowner_team: "@googleapis/analytics-dpe" - excluded_poms: proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1 - excluded_dependencies: google-iam-policy - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - - api_shortname: another-cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - GAPICs: - - proto_path: google/cloud/asset/v1 diff --git a/hermetic_build/common/tests/resources/test-config/monorepo_baseline.yaml b/hermetic_build/common/tests/resources/test-config/monorepo_baseline.yaml deleted file mode 100644 index c2c4fd4a3b..0000000000 --- a/hermetic_build/common/tests/resources/test-config/monorepo_baseline.yaml +++ /dev/null @@ -1,30 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: preview - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - api_shortname: cloudbuild - name_pretty: Cloud Build - product_documentation: https://cloud.google.com/cloud-build/ - api_description: lets you build software quickly across all languages. Get complete - control over defining custom workflows for building, testing, and deploying across - multiple environments such as VMs, serverless, Kubernetes, or Firebase. - release_level: stable - distribution_name: com.google.cloud:google-cloud-build - issue_tracker: https://issuetracker.google.com/savedsearches/5226584 - GAPICs: - - proto_path: google/devtools/cloudbuild/v1 - - proto_path: google/devtools/cloudbuild/v2 diff --git a/hermetic_build/common/tests/resources/test-config/monorepo_current.yaml b/hermetic_build/common/tests/resources/test-config/monorepo_current.yaml deleted file mode 100644 index 3ee2c8be2c..0000000000 --- a/hermetic_build/common/tests/resources/test-config/monorepo_current.yaml +++ /dev/null @@ -1,30 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: stable - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - api_shortname: cloudbuild - name_pretty: Cloud Build - product_documentation: https://cloud.google.com/cloud-build/ - api_description: lets you build software quickly across all languages. Get complete - control over defining custom workflows for building, testing, and deploying across - multiple environments such as VMs, serverless, Kubernetes, or Firebase. - release_level: stable - distribution_name: com.google.cloud:google-cloud-build - issue_tracker: https://issuetracker.google.com/savedsearches/5226584 - GAPICs: - - proto_path: google/devtools/cloudbuild/v1 - - proto_path: google/devtools/cloudbuild/v2 diff --git a/hermetic_build/common/tests/resources/test-config/monorepo_with_common_protos.yaml b/hermetic_build/common/tests/resources/test-config/monorepo_with_common_protos.yaml deleted file mode 100644 index 6d4c94444a..0000000000 --- a/hermetic_build/common/tests/resources/test-config/monorepo_with_common_protos.yaml +++ /dev/null @@ -1,43 +0,0 @@ -googleapis_commitish: 6a474b31c53cc1797710206824a17b364a835d2d -gapic_generator_version: 2.34.0 -# the libraries are ordered with respect to library name, which is -# java-{library.library_name} or java-{library.api-shortname} when -# library.library_name is not defined. -libraries: -- api_shortname: common-protos - name_pretty: Common Protos - product_documentation: https://github.com/googleapis/api-common-protos - api_description: Protobuf classes for Google's common protos. - release_level: stable - client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history - distribution_name: com.google.api.grpc:proto-google-common-protos - excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" - excluded_poms: "proto-google-common-protos-bom,proto-google-common-protos" - library_type: OTHER - GAPICs: - - proto_path: google/api - - proto_path: google/apps/card/v1 - - proto_path: google/cloud - - proto_path: google/cloud/audit - - proto_path: google/cloud/location - - proto_path: google/geo/type - - proto_path: google/logging/type - - proto_path: google/longrunning - - proto_path: google/rpc - - proto_path: google/rpc/context - - proto_path: google/shopping/type - - proto_path: google/type -- api_shortname: iam - name_pretty: IAM - product_documentation: https://cloud.google.com/iam - api_description: Manages access control for Google Cloud Platform resources - release_level: stable - client_documentation: https://cloud.google.com/java/docs/reference/proto-google-iam-v1/latest/overview - distribution_name: com.google.api.grpc:proto-google-iam-v1 - excluded_dependencies: "grpc-google-iam-v1" - excluded_poms: "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1" - library_type: OTHER - GAPICs: - - proto_path: google/iam/v1 - - proto_path: google/iam/v2 - - proto_path: google/iam/v2beta \ No newline at end of file diff --git a/hermetic_build/common/tests/resources/test-config/monorepo_without_common_protos.yaml b/hermetic_build/common/tests/resources/test-config/monorepo_without_common_protos.yaml deleted file mode 100644 index ca21ccfb01..0000000000 --- a/hermetic_build/common/tests/resources/test-config/monorepo_without_common_protos.yaml +++ /dev/null @@ -1,33 +0,0 @@ -gapic_generator_version: 2.34.0 -protoc_version: 25.2 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: cloudasset - name_pretty: Cloud Asset Inventory - product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - api_description: "provides inventory services based on a time series database." - library_name: "asset" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" - distribution_name: "com.google.cloud:google-cloud-asset" - release_level: "stable" - issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" - api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" - GAPICs: - - proto_path: google/cloud/asset/v1 - - proto_path: google/cloud/asset/v1p1beta1 - - proto_path: google/cloud/asset/v1p2beta1 - - proto_path: google/cloud/asset/v1p5beta1 - - proto_path: google/cloud/asset/v1p7beta1 - - api_shortname: cloudbuild - name_pretty: Cloud Build - product_documentation: https://cloud.google.com/cloud-build/ - api_description: lets you build software quickly across all languages. Get complete - control over defining custom workflows for building, testing, and deploying across - multiple environments such as VMs, serverless, Kubernetes, or Firebase. - release_level: stable - distribution_name: com.google.cloud:google-cloud-build - issue_tracker: https://issuetracker.google.com/savedsearches/5226584 - GAPICs: - - proto_path: google/devtools/cloudbuild/v1 - - proto_path: google/devtools/cloudbuild/v2 diff --git a/hermetic_build/library_generation/tests/resources/goldens/pr_description-golden.txt b/hermetic_build/library_generation/tests/resources/goldens/pr_description-golden.txt deleted file mode 100644 index 1a0f874936..0000000000 --- a/hermetic_build/library_generation/tests/resources/goldens/pr_description-golden.txt +++ /dev/null @@ -1,17 +0,0 @@ -This pull request is generated with proto changes between [googleapis/googleapis@3b6f144](https://github.com/googleapis/googleapis/commit/3b6f144d47b0a1d2115ab2445ec06e80cc324a44) (exclusive) and [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) (inclusive). - -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -chore: update the libraries_bom version to 2.3.4 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -feat: Make Layout Parser generally available in V1 - -PiperOrigin-RevId: 638924855 - -Source Link: [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt b/hermetic_build/library_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt deleted file mode 100644 index 81f7c0c4d2..0000000000 --- a/hermetic_build/library_generation/tests/resources/goldens/repo_level_and_no_qualified_commit_pr_description-golden.txt +++ /dev/null @@ -1,10 +0,0 @@ -This pull request is generated with proto changes between [googleapis/googleapis@3b6f144](https://github.com/googleapis/googleapis/commit/3b6f144d47b0a1d2115ab2445ec06e80cc324a44) (exclusive) and [googleapis/googleapis@0cea717](https://github.com/googleapis/googleapis/commit/0cea7170404bec3d994f43db4fa292f5034cbe9a) (inclusive). - -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 -END_NESTED_COMMIT -BEGIN_NESTED_COMMIT -chore: update the libraries_bom version to 2.3.4 -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt b/hermetic_build/library_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt deleted file mode 100644 index f89901f8c1..0000000000 --- a/hermetic_build/library_generation/tests/resources/goldens/repo_level_only_pr_description-golden.txt +++ /dev/null @@ -1,5 +0,0 @@ -BEGIN_COMMIT_OVERRIDE -BEGIN_NESTED_COMMIT -fix(deps): update the Java code generator (gapic-generator-java) to 1.2.3 -END_NESTED_COMMIT -END_COMMIT_OVERRIDE \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_common_resources.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_common_resources.bazel deleted file mode 100644 index 126ffdb7ca..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_common_resources.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - #"//google/cloud:common_resources_proto", - ] -) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_iam_policy.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_iam_policy.bazel deleted file mode 100644 index a9a2c1ca75..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_iam_policy.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - # "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_locations.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_locations.bazel deleted file mode 100644 index 8b96e3ab81..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_comment_locations.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - # "//google/cloud/location:location_proto", - ] -) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_common_resources.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_common_resources.bazel deleted file mode 100644 index 9b749e6ad5..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_common_resources.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/cloud:common_resources_proto", - ] -) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_gapic_yaml.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_gapic_yaml.bazel deleted file mode 100644 index b55f4550d8..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_gapic_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - gapic_yaml = "test_gapic_yaml.yaml", -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc.bazel deleted file mode 100644 index 3e59a9bd35..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_transport_from_BUILD` in utilities.sh - -java_gapic_library( - transport = "grpc", -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc_rest.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc_rest.bazel deleted file mode 100644 index 8a98e7e646..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_grpc_rest.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_transport_from_BUILD` in utilities.sh - -java_gapic_library( - transport = "grpc+rest", -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_locations.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_locations.bazel deleted file mode 100644 index d0c971da7c..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_locations.bazel +++ /dev/null @@ -1,6 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/cloud/location:location_proto", - "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_policy.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_policy.bazel deleted file mode 100644 index af5d4a32f8..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_iam_policy.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_empty.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_empty.bazel deleted file mode 100644 index b9d4cd56e0..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_empty.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh - -java_gapic_assembly_gradle_pkg( - -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_false.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_false.bazel deleted file mode 100644 index 8e95c3d2a6..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_false.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh - -java_gapic_assembly_gradle_pkg( - include_samples = False, -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_true.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_true.bazel deleted file mode 100644 index bac72b678f..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_include_samples_true.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_include_samples_from_BUILD` in utilities.sh - -java_gapic_assembly_gradle_pkg( - include_samples = True, -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_locations.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_locations.bazel deleted file mode 100644 index 29ee14fdba..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_locations.bazel +++ /dev/null @@ -1,5 +0,0 @@ -proto_library_with_info( - deps = [ - "//google/cloud/location:location_proto", - ] -) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_additional_protos.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_additional_protos.bazel deleted file mode 100644 index a22257cad4..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_additional_protos.bazel +++ /dev/null @@ -1,4 +0,0 @@ -proto_library_with_info( - deps = [ - ] -) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_gapic_yaml.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_gapic_yaml.bazel deleted file mode 100644 index 1e9462aa30..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_gapic_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - gapic_yaml = None -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_config.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_config.bazel deleted file mode 100644 index dbde6de05c..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_config.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - grpc_service_config = None -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_yaml.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_yaml.bazel deleted file mode 100644 index 05bae16d5d..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_no_service_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - service_yaml = None -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_proto_only.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_proto_only.bazel deleted file mode 100644 index 26bcea6126..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_proto_only.bazel +++ /dev/null @@ -1,16 +0,0 @@ -java_gapic_assembly_gradle_pkg( - name = "google-api-java", - transport = "grpc+rest", - deps = [ - "annotations_proto", - "auth_proto", - "backend_proto", - "billing_proto", - "client_proto", - "config_change_proto", - "consumer_proto", - "context_proto", - "control_proto", - "distribution_proto", - ], -) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest.bazel deleted file mode 100644 index 9dff694297..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_transport_from_BUILD` in utilities.sh - -java_gapic_library( - transport = "rest", -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel deleted file mode 100644 index 992b91e52c..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_empty.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh - -java_gapic_library( - -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel deleted file mode 100644 index a446c6b2a7..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_false.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh - -java_gapic_library( - rest_numeric_enums = False -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel deleted file mode 100644 index c4d7fefeb4..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_rest_numeric_enums_true.bazel +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `get_rest_numeric_enums_from_BUILD` in utilities.sh - -java_gapic_library( - rest_numeric_enums = True -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config.bazel deleted file mode 100644 index 097d1bb6bd..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - grpc_service_config = "test_service_config.json" -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config_relative_target.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config_relative_target.bazel deleted file mode 100644 index ccd59af2fb..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_config_relative_target.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - grpc_service_config = ":compute_grpc_service_config.json" -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml.bazel deleted file mode 100644 index f7e4c91f4e..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - service_yaml = "test_service_yaml.yaml" -) diff --git a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel b/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel deleted file mode 100644 index ded899dff7..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/BUILD_service_yaml_absolute_target.bazel +++ /dev/null @@ -1,3 +0,0 @@ -java_gapic_library( - service_yaml = "//google/cloud/videointelligence:videointelligence_v1p3beta1.yaml", -) \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_api_description.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_api_description.yaml deleted file mode 100644 index 79ff135067..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_api_description.yaml +++ /dev/null @@ -1,5 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_api_shortname.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_api_shortname.yaml deleted file mode 100644 index ec8206be61..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_api_shortname.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_key.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_key.yaml deleted file mode 100644 index 739a4d9239..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_key.yaml +++ /dev/null @@ -1,3 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_value.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_value.yaml deleted file mode 100644 index ec49e4a669..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_gapics_value.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - GAPICs: diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_googleapis.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_googleapis.yaml deleted file mode 100644 index e5a00ca4ee..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_googleapis.yaml +++ /dev/null @@ -1,8 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_libraries.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_libraries.yaml deleted file mode 100644 index dbbe2ea318..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_libraries.yaml +++ /dev/null @@ -1 +0,0 @@ -gapic_generator_version: 2.34.0 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_library_value.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_library_value.yaml deleted file mode 100644 index 174a293000..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_library_value.yaml +++ /dev/null @@ -1,2 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_name_pretty.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_name_pretty.yaml deleted file mode 100644 index f8612ad9ca..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_name_pretty.yaml +++ /dev/null @@ -1,6 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - api_description: "allows the Apigee hybrid management" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_product_docs.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_product_docs.yaml deleted file mode 100644 index e3921d2c0d..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_product_docs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_proto_path.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_proto_path.yaml deleted file mode 100644 index e37b0cef63..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_proto_path.yaml +++ /dev/null @@ -1,4 +0,0 @@ -gapic_generator_version: 2.34.0 -libraries: - - GAPICs: - - random_key: diff --git a/hermetic_build/library_generation/tests/resources/test-config/config_without_temp_excludes.yaml b/hermetic_build/library_generation/tests/resources/test-config/config_without_temp_excludes.yaml deleted file mode 100644 index 0d1bb7deea..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-config/config_without_temp_excludes.yaml +++ /dev/null @@ -1,10 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries_bom_version: 26.37.0 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/hermetic_build/library_generation/tests/resources/test-monorepo/.github/.OwlBot.lock.yaml b/hermetic_build/library_generation/tests/resources/test-monorepo/.github/.OwlBot.lock.yaml deleted file mode 100644 index 77200af4c9..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-monorepo/.github/.OwlBot.lock.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -docker: - image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:fb7584f6adb3847ac480ed49a4bfe1463965026b2919a1be270e3174f3ce1191 - # created: 2023-01-20T00:00:00.000000000Z diff --git a/hermetic_build/library_generation/tests/resources/test-monorepo/test-service/.repo-metadata.json b/hermetic_build/library_generation/tests/resources/test-monorepo/test-service/.repo-metadata.json deleted file mode 100644 index d5b5078213..0000000000 --- a/hermetic_build/library_generation/tests/resources/test-monorepo/test-service/.repo-metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "api_shortname": "cloudasset", - "name_pretty": "Cloud Asset Inventory", - "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview", - "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview", - "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0", - "release_level": "stable", - "transport": "grpc", - "requires_billing": true, - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-asset", - "distribution_name": "com.google.cloud:google-cloud-asset", - "api_id": "cloudasset.googleapis.com", - "library_type": "GAPIC_AUTO" -} diff --git a/hermetic_build/library_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml b/hermetic_build/library_generation/tests/resources/test_generate_release_note/empty_gen_config.yaml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml b/hermetic_build/release_note_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml deleted file mode 100644 index 225b4620bf..0000000000 --- a/hermetic_build/release_note_generation/tests/resources/goldens/.OwlBot-hermetic-golden.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -deep-remove-regex: -- "/java-bare-metal-solution/grpc-google-.*/src" -- "/java-bare-metal-solution/proto-google-.*/src" -- "/java-bare-metal-solution/google-.*/src" -- "/java-bare-metal-solution/samples/snippets/generated" - -deep-preserve-regex: -- "/java-bare-metal-solution/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" - -deep-copy-regex: -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/proto-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/proto-google-cloud-bare-metal-solution-$1/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/grpc-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/grpc-google-cloud-bare-metal-solution-$1/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/gapic-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/google-cloud-bare-metal-solution/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/samples/snippets/generated" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/samples/snippets/generated" - -api-name: baremetalsolution \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json deleted file mode 100644 index 2cccd4b889..0000000000 --- a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-custom-transport-golden.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "api_shortname": "secretmanager", - "name_pretty": "Secret Management", - "product_documentation": "https://cloud.google.com/solutions/secrets-management/", - "api_description": "allows you to encrypt, store, manage, and audit infrastructure and application-level secrets.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-secretmanager/latest/overview", - "release_level": "preview", - "transport": "http", - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-secretmanager", - "distribution_name": "com.google.cloud:google-cloud-secretmanager", - "api_id": "secretmanager.googleapis.com", - "library_type": "GAPIC_AUTO", - "requires_billing": true -} \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json deleted file mode 100644 index 0b3bbc2b55..0000000000 --- a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-monorepo-golden.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "api_id": "baremetalsolution.googleapis.com", - "library_type": "GAPIC_AUTO", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", - "recommended_package": "com.google.example", - "min_java_version": 8 -} \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json deleted file mode 100644 index b39c297a85..0000000000 --- a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-non-monorepo-golden.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/java-bare-metal-solution", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "api_id": "baremetalsolution.googleapis.com", - "library_type": "GAPIC_COMBO", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", - "extra_versioned_modules": "test-module", - "recommended_package": "com.google.example", - "min_java_version": 8 -} \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json b/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json deleted file mode 100644 index 7730cd6987..0000000000 --- a/hermetic_build/release_note_generation/tests/resources/goldens/.repo-metadata-proto-only-golden.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/sdk-platform-java", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "library_type": "OTHER", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", - "recommended_package": "com.google.example", - "min_java_version": 8 -} \ No newline at end of file diff --git a/hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py b/hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py deleted file mode 100644 index 7559eaf034..0000000000 --- a/hermetic_build/release_note_generation/tests/resources/goldens/owlbot-golden.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import synthtool as s -from synthtool.languages import java - - -for library in s.get_staging_dirs(): - # put any special-case replacements here - s.move(library) - -s.remove_staging_dirs() -java.common_templates( - monorepo=True, - excludes=[ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "SECURITY.md", - "java.header", - "license-checks.xml", - "renovate.json", - ".gitignore", - ], -) From 66e3812c600197726e0b8d7a1aa10c38fbcf1a5d Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 11:48:03 -0400 Subject: [PATCH 59/69] remove unused test resources --- .../tests/resources/misc/TESTWORKSPACE | 133 ------------------ 1 file changed, 133 deletions(-) delete mode 100644 hermetic_build/library_generation/tests/resources/misc/TESTWORKSPACE diff --git a/hermetic_build/library_generation/tests/resources/misc/TESTWORKSPACE b/hermetic_build/library_generation/tests/resources/misc/TESTWORKSPACE deleted file mode 100644 index 60b3036d9d..0000000000 --- a/hermetic_build/library_generation/tests/resources/misc/TESTWORKSPACE +++ /dev/null @@ -1,133 +0,0 @@ -# test workspace file obtained from sdk-platform-java - -workspace(name = "gapic_generator_java") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -# gax-java and its transitive dependencies must be imported before -# gapic-generator-java dependencies to match the order in googleapis repository, -# which in its turn, prioritizes actual generated clients runtime dependencies -# over the generator dependencies. -local_repository( - name = "com_google_api_gax_java", - path = "gax-java", -) - -load("@com_google_api_gax_java//:repository_rules.bzl", "com_google_api_gax_java_properties") - -com_google_api_gax_java_properties( - name = "com_google_api_gax_java_properties", - file = "@com_google_api_gax_java//:dependencies.properties", -) - -load("@com_google_api_gax_java//:repositories.bzl", "com_google_api_gax_java_repositories") - -com_google_api_gax_java_repositories() - -_googleapis_commit = "7438480b2a1bc6371d748e974f7a3647f90c4e8d" - -http_archive( - name = "com_google_googleapis", - strip_prefix = "googleapis-%s" % _googleapis_commit, - urls = [ - "https://github.com/googleapis/googleapis/archive/%s.zip" % _googleapis_commit, - ], -) - -# protobuf -RULES_JVM_EXTERNAL_TAG = "4.5" - -RULES_JVM_EXTERNAL_SHA = "b17d7388feb9bfa7f2fa09031b32707df529f26c91ab9e5d909eb1676badd9a6" - -http_archive( - name = "rules_jvm_external", - sha256 = RULES_JVM_EXTERNAL_SHA, - strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, - url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, -) - -load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") - -rules_jvm_external_deps() - -load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") - -rules_jvm_external_setup() - -load("@com_google_protobuf//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") -load("@rules_jvm_external//:defs.bzl", "maven_install") - -maven_install( - artifacts = PROTOBUF_MAVEN_ARTIFACTS, - repositories = ["https://repo.maven.apache.org/maven2/"], -) - -_gapic_generator_java_version = "2.25.1-SNAPSHOT" # {x-version-update:gapic-generator-java:current} - -maven_install( - artifacts = [ - "com.google.api:gapic-generator-java:" + _gapic_generator_java_version, - ], - fail_on_missing_checksum = False, - repositories = [ - "m2Local", - "https://repo.maven.apache.org/maven2/", - ], -) - -protobuf_deps() - -# Bazel rules. -_rules_gapic_version = "0.5.5" - -http_archive( - name = "rules_gapic", - strip_prefix = "rules_gapic-%s" % _rules_gapic_version, - urls = ["https://github.com/googleapis/rules_gapic/archive/v%s.tar.gz" % _rules_gapic_version], -) - -# Java dependencies. -load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") - -switched_rules_by_language( - name = "com_google_googleapis_imports", - gapic = True, - grpc = True, - java = True, -) - -load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories") - -grpc_java_repositories() - -_disco_to_proto3_converter_commit = "ce8d8732120cdfb5bf4847c3238b5be8acde87e3" - -http_archive( - name = "com_google_disco_to_proto3_converter", - strip_prefix = "disco-to-proto3-converter-%s" % _disco_to_proto3_converter_commit, - urls = ["https://github.com/googleapis/disco-to-proto3-converter/archive/%s.zip" % _disco_to_proto3_converter_commit], -) - -# Showcase -_showcase_version = "0.28.2" - -http_archive( - name = "com_google_gapic_showcase", - strip_prefix = "gapic-showcase-%s" % _showcase_version, - urls = [ - "https://github.com/googleapis/gapic-showcase/archive/refs/tags/v%s.zip" % _showcase_version, - ], -) - -http_archive( - name = "rules_pkg", - sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", - "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", - ], -) - -load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") - -rules_pkg_dependencies() From af986fbb6506feb59b99460bf9a7adb588c3f607 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 21 Oct 2024 11:55:46 -0400 Subject: [PATCH 60/69] change doc --- .../library_generation/DEVELOPMENT.md | 5 ++-- hermetic_build/library_generation/README.md | 28 ++++++++----------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/hermetic_build/library_generation/DEVELOPMENT.md b/hermetic_build/library_generation/DEVELOPMENT.md index f32b544e1e..51cfef7f6c 100644 --- a/hermetic_build/library_generation/DEVELOPMENT.md +++ b/hermetic_build/library_generation/DEVELOPMENT.md @@ -22,8 +22,7 @@ It requires docker and python 3.x to be installed. ``` python -m pip install . -python -m pip install -r requirements.txt -python -m unittest test/integration_tests.py +python -m unittest tests/integration_tests.py ``` # Running the unit tests @@ -34,7 +33,7 @@ Every unit test script ends with `unit_tests.py`. To avoid them specifying them individually, we can use the following command: ```bash -python -m unittest discover -s test/ -p "*unit_tests.py" +python -m unittest discover -s tests/ -p "*unit_tests.py" ``` > [!NOTE] diff --git a/hermetic_build/library_generation/README.md b/hermetic_build/library_generation/README.md index 518d588f9a..ede5a68e61 100644 --- a/hermetic_build/library_generation/README.md +++ b/hermetic_build/library_generation/README.md @@ -8,7 +8,7 @@ configuration file. - OS: Linux - Java runtime environment (8 or above) -- Python (3.11.6 or above) +- Python (3.12 or above) - Docker - Git @@ -190,27 +190,23 @@ libraries: 1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case). 2. Assuming the virtual environment is installed under `sdk-platform-java`. -Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation` +Run the following command under the root folder of `sdk-platform-java` to install `library_generation` and its dependencies. ```bash - python -m pip install --require-hashes -r library_generation/requirements.txt + python -m pip install hermetic_build/common + python -m pip install hermetic_build/library_generation ``` -3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere - ```bash - python -m pip install library_generation/ - ``` - -4. Download api definition to a local directory +3. Download api definition to a local directory ## An example to generate a repository using `entry_point.py` ```bash -python library_generation/entry_point.py generate \ ---baseline-generation-config-path=/path/to/baseline_config_file \ ---current-generation-config-path=/path/to/current_config_file \ ---repository-path=path/to/repository \ ---api-definitions-path=path/to/api_definition +python hermetic_build/library_generation/cli/entry_point.py generate \ + --baseline-generation-config-path=/path/to/baseline_config_file \ + --current-generation-config-path=/path/to/current_config_file \ + --repository-path=path/to/repository \ + --api-definitions-path=path/to/api_definition ``` If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration) shown above, the repository structure is: @@ -290,9 +286,7 @@ $repository_path We have transferred the [implementation](https://github.com/googleapis/synthtool/tree/59fe44fde9866a26e7ee4e4450fd79f67f8cf599/docker/owlbot/java) -of Java Owlbot Postprocessor into `sdk-platform-java/library_generation`. The -implementation in synthtool is still valid and used by other services, so we -have two versions during a transition period. +of Java Owlbot Postprocessor into `sdk-platform-java/hermetic_build/library_generation`. ## Reflecting changes in synthtool/docker/owlbot/java into this repository The transfer was not a verbatim copy, it rather had modifications: From 1c9a9a7b14b84865bba2ccbf05ca57a20b3402c7 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 22 Oct 2024 15:55:08 -0400 Subject: [PATCH 61/69] create requirements.in for each module --- .github/scripts/action.yaml | 4 + .../workflows/verify_library_generation.yaml | 8 + hermetic_build/common/requirements.in | 3 + hermetic_build/common/requirements.txt | 109 +++++ hermetic_build/common/setup.py | 7 +- .../library_generation/requirements.in | 8 + .../library_generation/requirements.txt | 417 ++++++++++++++++++ hermetic_build/library_generation/setup.py | 11 - .../release_note_generation/requirements.in | 2 + .../release_note_generation/requirements.txt | 22 + .../release_note_generation/setup.py | 8 +- 11 files changed, 575 insertions(+), 24 deletions(-) create mode 100644 hermetic_build/common/requirements.in create mode 100644 hermetic_build/common/requirements.txt create mode 100644 hermetic_build/library_generation/requirements.in create mode 100644 hermetic_build/library_generation/requirements.txt create mode 100644 hermetic_build/release_note_generation/requirements.in create mode 100644 hermetic_build/release_note_generation/requirements.txt diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 447c17e25a..6bbfdcb5a4 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -58,8 +58,12 @@ runs: shell: bash run: | cd "${GITHUB_WORKSPACE}" + pip install --upgrade pip + pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common + pip install --require-hashes -r hermetic_build/library_generation/requirements.txt pip install hermetic_build/library_generation + pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt pip install hermetic_build/release_note_generation - name: Generate changed libraries shell: bash diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index 8c12704b11..f7854a9539 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -52,7 +52,9 @@ jobs: run: | set -ex pip install --upgrade pip + pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common + pip install --require-hashes -r hermetic_build/library_generation/requirements.txt pip install hermetic_build/library_generation - name: Run integration tests shell: bash @@ -73,8 +75,11 @@ jobs: run: | set -ex pip install --upgrade pip + pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common + pip install --require-hashes -r hermetic_build/library_generation/requirements.txt pip install hermetic_build/library_generation + pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt pip install hermetic_build/release_note_generation - name: Run shell unit tests run: | @@ -110,8 +115,11 @@ jobs: run: | set -ex pip install --upgrade pip + pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common + pip install --require-hashes -r hermetic_build/library_generation/requirements.txt pip install hermetic_build/library_generation + pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt pip install hermetic_build/release_note_generation - name: Lint shell: bash diff --git a/hermetic_build/common/requirements.in b/hermetic_build/common/requirements.in new file mode 100644 index 0000000000..a34205e5fa --- /dev/null +++ b/hermetic_build/common/requirements.in @@ -0,0 +1,3 @@ +black==24.8.0 +parameterized==0.9.0 +PyYAML==6.0.2 \ No newline at end of file diff --git a/hermetic_build/common/requirements.txt b/hermetic_build/common/requirements.txt new file mode 100644 index 0000000000..9253e5ed5a --- /dev/null +++ b/hermetic_build/common/requirements.txt @@ -0,0 +1,109 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --generate-hashes hermetic_build/common/requirements.in +# +black==24.8.0 \ + --hash=sha256:09cdeb74d494ec023ded657f7092ba518e8cf78fa8386155e4a03fdcc44679e6 \ + --hash=sha256:1f13f7f386f86f8121d76599114bb8c17b69d962137fc70efe56137727c7047e \ + --hash=sha256:2500945420b6784c38b9ee885af039f5e7471ef284ab03fa35ecdde4688cd83f \ + --hash=sha256:2b59b250fdba5f9a9cd9d0ece6e6d993d91ce877d121d161e4698af3eb9c1018 \ + --hash=sha256:3c4285573d4897a7610054af5a890bde7c65cb466040c5f0c8b732812d7f0e5e \ + --hash=sha256:505289f17ceda596658ae81b61ebbe2d9b25aa78067035184ed0a9d855d18afd \ + --hash=sha256:62e8730977f0b77998029da7971fa896ceefa2c4c4933fcd593fa599ecbf97a4 \ + --hash=sha256:649f6d84ccbae73ab767e206772cc2d7a393a001070a4c814a546afd0d423aed \ + --hash=sha256:6e55d30d44bed36593c3163b9bc63bf58b3b30e4611e4d88a0c3c239930ed5b2 \ + --hash=sha256:707a1ca89221bc8a1a64fb5e15ef39cd755633daa672a9db7498d1c19de66a42 \ + --hash=sha256:72901b4913cbac8972ad911dc4098d5753704d1f3c56e44ae8dce99eecb0e3af \ + --hash=sha256:73bbf84ed136e45d451a260c6b73ed674652f90a2b3211d6a35e78054563a9bb \ + --hash=sha256:7c046c1d1eeb7aea9335da62472481d3bbf3fd986e093cffd35f4385c94ae368 \ + --hash=sha256:81c6742da39f33b08e791da38410f32e27d632260e599df7245cccee2064afeb \ + --hash=sha256:837fd281f1908d0076844bc2b801ad2d369c78c45cf800cad7b61686051041af \ + --hash=sha256:972085c618ee94f402da1af548a4f218c754ea7e5dc70acb168bfaca4c2542ed \ + --hash=sha256:9e84e33b37be070ba135176c123ae52a51f82306def9f7d063ee302ecab2cf47 \ + --hash=sha256:b19c9ad992c7883ad84c9b22aaa73562a16b819c1d8db7a1a1a49fb7ec13c7d2 \ + --hash=sha256:d6417535d99c37cee4091a2f24eb2b6d5ec42b144d50f1f2e436d9fe1916fe1a \ + --hash=sha256:eab4dd44ce80dea27dc69db40dab62d4ca96112f87996bca68cd75639aeb2e4c \ + --hash=sha256:f490dbd59680d809ca31efdae20e634f3fae27fba3ce0ba3208333b713bc3920 \ + --hash=sha256:fb6e2c0b86bbd43dee042e48059c9ad7830abd5c94b0bc518c0eeec57c3eddc1 + # via -r hermetic_build/common/requirements.in +click==8.1.7 \ + --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ + --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de + # via black +mypy-extensions==1.0.0 \ + --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ + --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 + # via black +packaging==24.1 \ + --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ + --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 + # via black +parameterized==0.9.0 \ + --hash=sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b \ + --hash=sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1 + # via -r hermetic_build/common/requirements.in +pathspec==0.12.1 \ + --hash=sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 \ + --hash=sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712 + # via black +platformdirs==4.3.6 \ + --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ + --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb + # via black +pyyaml==6.0.2 \ + --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \ + --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ + --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \ + --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \ + --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ + --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \ + --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ + --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \ + --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ + --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \ + --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \ + --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \ + --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \ + --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \ + --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \ + --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \ + --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ + --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \ + --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ + --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \ + --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \ + --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \ + --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \ + --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ + --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ + --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \ + --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \ + --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ + --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ + --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \ + --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ + --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \ + --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ + --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \ + --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \ + --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ + --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \ + --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \ + --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \ + --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ + --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ + --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \ + --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ + --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ + --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \ + --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \ + --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \ + --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \ + --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \ + --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \ + --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \ + --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ + --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 + # via -r hermetic_build/common/requirements.in diff --git a/hermetic_build/common/setup.py b/hermetic_build/common/setup.py index c3c69ce59f..2eddf36d0b 100755 --- a/hermetic_build/common/setup.py +++ b/hermetic_build/common/setup.py @@ -10,10 +10,5 @@ python_requires=">=3.12", package_dir={ "common": ".", - }, - install_requires=[ - "black==24.8.0", - "parameterized==0.9.0", - "PyYAML==6.0.2", - ], + } ) diff --git a/hermetic_build/library_generation/requirements.in b/hermetic_build/library_generation/requirements.in new file mode 100644 index 0000000000..4f8ad9709b --- /dev/null +++ b/hermetic_build/library_generation/requirements.in @@ -0,0 +1,8 @@ +attrs==24.2.0 +click==8.1.7 +GitPython==3.1.43 +jinja2==3.1.4 +lxml==5.3.0 +PyYAML==6.0.2 +requests==2.32.3 +requests-mock==1.12.1 \ No newline at end of file diff --git a/hermetic_build/library_generation/requirements.txt b/hermetic_build/library_generation/requirements.txt new file mode 100644 index 0000000000..e3ed826ac7 --- /dev/null +++ b/hermetic_build/library_generation/requirements.txt @@ -0,0 +1,417 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --generate-hashes hermetic_build/library_generation/requirements.in +# +attrs==24.2.0 \ + --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \ + --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2 + # via -r hermetic_build/library_generation/requirements.in +certifi==2024.8.30 \ + --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \ + --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9 + # via requests +charset-normalizer==3.4.0 \ + --hash=sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621 \ + --hash=sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6 \ + --hash=sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8 \ + --hash=sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912 \ + --hash=sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c \ + --hash=sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b \ + --hash=sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d \ + --hash=sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d \ + --hash=sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95 \ + --hash=sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e \ + --hash=sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565 \ + --hash=sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64 \ + --hash=sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab \ + --hash=sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be \ + --hash=sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e \ + --hash=sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907 \ + --hash=sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0 \ + --hash=sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2 \ + --hash=sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62 \ + --hash=sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62 \ + --hash=sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23 \ + --hash=sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc \ + --hash=sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284 \ + --hash=sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca \ + --hash=sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455 \ + --hash=sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858 \ + --hash=sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b \ + --hash=sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594 \ + --hash=sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc \ + --hash=sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db \ + --hash=sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b \ + --hash=sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea \ + --hash=sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6 \ + --hash=sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920 \ + --hash=sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749 \ + --hash=sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7 \ + --hash=sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd \ + --hash=sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99 \ + --hash=sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242 \ + --hash=sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee \ + --hash=sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129 \ + --hash=sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2 \ + --hash=sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51 \ + --hash=sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee \ + --hash=sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8 \ + --hash=sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b \ + --hash=sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613 \ + --hash=sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742 \ + --hash=sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe \ + --hash=sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3 \ + --hash=sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5 \ + --hash=sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631 \ + --hash=sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7 \ + --hash=sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15 \ + --hash=sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c \ + --hash=sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea \ + --hash=sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417 \ + --hash=sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250 \ + --hash=sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88 \ + --hash=sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca \ + --hash=sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa \ + --hash=sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99 \ + --hash=sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149 \ + --hash=sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41 \ + --hash=sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574 \ + --hash=sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0 \ + --hash=sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f \ + --hash=sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d \ + --hash=sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654 \ + --hash=sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3 \ + --hash=sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19 \ + --hash=sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90 \ + --hash=sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578 \ + --hash=sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9 \ + --hash=sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1 \ + --hash=sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51 \ + --hash=sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719 \ + --hash=sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236 \ + --hash=sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a \ + --hash=sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c \ + --hash=sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade \ + --hash=sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944 \ + --hash=sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc \ + --hash=sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6 \ + --hash=sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6 \ + --hash=sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27 \ + --hash=sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6 \ + --hash=sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2 \ + --hash=sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12 \ + --hash=sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf \ + --hash=sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114 \ + --hash=sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7 \ + --hash=sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf \ + --hash=sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d \ + --hash=sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b \ + --hash=sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed \ + --hash=sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03 \ + --hash=sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4 \ + --hash=sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67 \ + --hash=sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365 \ + --hash=sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a \ + --hash=sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748 \ + --hash=sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b \ + --hash=sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079 \ + --hash=sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482 + # via requests +click==8.1.7 \ + --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ + --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de + # via -r hermetic_build/library_generation/requirements.in +gitdb==4.0.11 \ + --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ + --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b + # via gitpython +gitpython==3.1.43 \ + --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ + --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff + # via -r hermetic_build/library_generation/requirements.in +idna==3.10 \ + --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ + --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 + # via requests +jinja2==3.1.4 \ + --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ + --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d + # via -r hermetic_build/library_generation/requirements.in +lxml==5.3.0 \ + --hash=sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e \ + --hash=sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229 \ + --hash=sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3 \ + --hash=sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5 \ + --hash=sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70 \ + --hash=sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15 \ + --hash=sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002 \ + --hash=sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd \ + --hash=sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22 \ + --hash=sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf \ + --hash=sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22 \ + --hash=sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832 \ + --hash=sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727 \ + --hash=sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e \ + --hash=sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30 \ + --hash=sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f \ + --hash=sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f \ + --hash=sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51 \ + --hash=sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4 \ + --hash=sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de \ + --hash=sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875 \ + --hash=sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42 \ + --hash=sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e \ + --hash=sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6 \ + --hash=sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391 \ + --hash=sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc \ + --hash=sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b \ + --hash=sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237 \ + --hash=sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4 \ + --hash=sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86 \ + --hash=sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f \ + --hash=sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a \ + --hash=sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8 \ + --hash=sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f \ + --hash=sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903 \ + --hash=sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03 \ + --hash=sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e \ + --hash=sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99 \ + --hash=sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7 \ + --hash=sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab \ + --hash=sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d \ + --hash=sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22 \ + --hash=sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492 \ + --hash=sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b \ + --hash=sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3 \ + --hash=sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be \ + --hash=sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469 \ + --hash=sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f \ + --hash=sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a \ + --hash=sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c \ + --hash=sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a \ + --hash=sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4 \ + --hash=sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94 \ + --hash=sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442 \ + --hash=sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b \ + --hash=sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84 \ + --hash=sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c \ + --hash=sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9 \ + --hash=sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1 \ + --hash=sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be \ + --hash=sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367 \ + --hash=sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e \ + --hash=sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21 \ + --hash=sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa \ + --hash=sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16 \ + --hash=sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d \ + --hash=sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe \ + --hash=sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83 \ + --hash=sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba \ + --hash=sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040 \ + --hash=sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763 \ + --hash=sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8 \ + --hash=sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff \ + --hash=sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2 \ + --hash=sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a \ + --hash=sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b \ + --hash=sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce \ + --hash=sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c \ + --hash=sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577 \ + --hash=sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8 \ + --hash=sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71 \ + --hash=sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512 \ + --hash=sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540 \ + --hash=sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f \ + --hash=sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2 \ + --hash=sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a \ + --hash=sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce \ + --hash=sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e \ + --hash=sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2 \ + --hash=sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27 \ + --hash=sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1 \ + --hash=sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d \ + --hash=sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1 \ + --hash=sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330 \ + --hash=sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920 \ + --hash=sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99 \ + --hash=sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff \ + --hash=sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18 \ + --hash=sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff \ + --hash=sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c \ + --hash=sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179 \ + --hash=sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080 \ + --hash=sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19 \ + --hash=sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d \ + --hash=sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70 \ + --hash=sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32 \ + --hash=sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a \ + --hash=sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2 \ + --hash=sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79 \ + --hash=sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3 \ + --hash=sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5 \ + --hash=sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f \ + --hash=sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d \ + --hash=sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3 \ + --hash=sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b \ + --hash=sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753 \ + --hash=sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9 \ + --hash=sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957 \ + --hash=sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033 \ + --hash=sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb \ + --hash=sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656 \ + --hash=sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab \ + --hash=sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b \ + --hash=sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d \ + --hash=sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd \ + --hash=sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859 \ + --hash=sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11 \ + --hash=sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c \ + --hash=sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a \ + --hash=sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005 \ + --hash=sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654 \ + --hash=sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80 \ + --hash=sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e \ + --hash=sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec \ + --hash=sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7 \ + --hash=sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965 \ + --hash=sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945 \ + --hash=sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8 + # via -r hermetic_build/library_generation/requirements.in +markupsafe==3.0.2 \ + --hash=sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4 \ + --hash=sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30 \ + --hash=sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0 \ + --hash=sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9 \ + --hash=sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396 \ + --hash=sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13 \ + --hash=sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028 \ + --hash=sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca \ + --hash=sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557 \ + --hash=sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832 \ + --hash=sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0 \ + --hash=sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b \ + --hash=sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579 \ + --hash=sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a \ + --hash=sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c \ + --hash=sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff \ + --hash=sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c \ + --hash=sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22 \ + --hash=sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094 \ + --hash=sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb \ + --hash=sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e \ + --hash=sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5 \ + --hash=sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a \ + --hash=sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d \ + --hash=sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a \ + --hash=sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b \ + --hash=sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8 \ + --hash=sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225 \ + --hash=sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c \ + --hash=sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144 \ + --hash=sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f \ + --hash=sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87 \ + --hash=sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d \ + --hash=sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93 \ + --hash=sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf \ + --hash=sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158 \ + --hash=sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84 \ + --hash=sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb \ + --hash=sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48 \ + --hash=sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171 \ + --hash=sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c \ + --hash=sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6 \ + --hash=sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd \ + --hash=sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d \ + --hash=sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1 \ + --hash=sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d \ + --hash=sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca \ + --hash=sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a \ + --hash=sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29 \ + --hash=sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe \ + --hash=sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798 \ + --hash=sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c \ + --hash=sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8 \ + --hash=sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f \ + --hash=sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f \ + --hash=sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a \ + --hash=sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178 \ + --hash=sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0 \ + --hash=sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79 \ + --hash=sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430 \ + --hash=sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50 + # via jinja2 +pyyaml==6.0.2 \ + --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \ + --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \ + --hash=sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086 \ + --hash=sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e \ + --hash=sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133 \ + --hash=sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5 \ + --hash=sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484 \ + --hash=sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee \ + --hash=sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5 \ + --hash=sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68 \ + --hash=sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a \ + --hash=sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf \ + --hash=sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99 \ + --hash=sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8 \ + --hash=sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85 \ + --hash=sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19 \ + --hash=sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc \ + --hash=sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a \ + --hash=sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1 \ + --hash=sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317 \ + --hash=sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c \ + --hash=sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631 \ + --hash=sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d \ + --hash=sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652 \ + --hash=sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5 \ + --hash=sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e \ + --hash=sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b \ + --hash=sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8 \ + --hash=sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 \ + --hash=sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706 \ + --hash=sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563 \ + --hash=sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237 \ + --hash=sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b \ + --hash=sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083 \ + --hash=sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180 \ + --hash=sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425 \ + --hash=sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e \ + --hash=sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f \ + --hash=sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725 \ + --hash=sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183 \ + --hash=sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab \ + --hash=sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774 \ + --hash=sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 \ + --hash=sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e \ + --hash=sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5 \ + --hash=sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d \ + --hash=sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290 \ + --hash=sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44 \ + --hash=sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed \ + --hash=sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4 \ + --hash=sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba \ + --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ + --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 + # via -r hermetic_build/library_generation/requirements.in +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 + # via + # -r hermetic_build/library_generation/requirements.in + # requests-mock +requests-mock==1.12.1 \ + --hash=sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563 \ + --hash=sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401 + # via -r hermetic_build/library_generation/requirements.in +smmap==5.0.1 \ + --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ + --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da + # via gitdb +urllib3==2.2.3 \ + --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \ + --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9 + # via requests diff --git a/hermetic_build/library_generation/setup.py b/hermetic_build/library_generation/setup.py index 19737b9890..d8bbb4845b 100755 --- a/hermetic_build/library_generation/setup.py +++ b/hermetic_build/library_generation/setup.py @@ -12,17 +12,6 @@ "library_generation": ".", "synthtool": "owlbot/synthtool", }, - install_requires=[ - "attrs==24.2.0", - "click==8.1.7", - "common==0.1", # local package - "GitPython==3.1.43", - "jinja2==3.1.4", - "lxml==5.3.0", - "PyYAML==6.0.2", - "requests==2.32.3", - "requests-mock==1.12.1", - ], package_data={ "library_generation": [ "generate_library.sh", diff --git a/hermetic_build/release_note_generation/requirements.in b/hermetic_build/release_note_generation/requirements.in new file mode 100644 index 0000000000..4c9d8e47d7 --- /dev/null +++ b/hermetic_build/release_note_generation/requirements.in @@ -0,0 +1,2 @@ +click==8.1.7 +GitPython==3.1.43 \ No newline at end of file diff --git a/hermetic_build/release_note_generation/requirements.txt b/hermetic_build/release_note_generation/requirements.txt new file mode 100644 index 0000000000..9eb391e0e2 --- /dev/null +++ b/hermetic_build/release_note_generation/requirements.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --generate-hashes hermetic_build/release_note_generation/requirements.in +# +click==8.1.7 \ + --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ + --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de + # via -r hermetic_build/release_note_generation/requirements.in +gitdb==4.0.11 \ + --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ + --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b + # via gitpython +gitpython==3.1.43 \ + --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ + --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff + # via -r hermetic_build/release_note_generation/requirements.in +smmap==5.0.1 \ + --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ + --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da + # via gitdb diff --git a/hermetic_build/release_note_generation/setup.py b/hermetic_build/release_note_generation/setup.py index 81628e9120..c7506b60d5 100755 --- a/hermetic_build/release_note_generation/setup.py +++ b/hermetic_build/release_note_generation/setup.py @@ -10,11 +10,5 @@ python_requires=">=3.12", package_dir={ "release_note_generation": ".", - }, - install_requires=[ - "click==8.1.7", - "common==0.1", # local package - "GitPython==3.1.43", - "PyYAML==6.0.2", - ], + } ) From 394733f7acaa272a37435c8e177112e8b3ba1198 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 22 Oct 2024 15:58:42 -0400 Subject: [PATCH 62/69] lint --- hermetic_build/common/setup.py | 2 +- hermetic_build/release_note_generation/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hermetic_build/common/setup.py b/hermetic_build/common/setup.py index 2eddf36d0b..8c3c663ada 100755 --- a/hermetic_build/common/setup.py +++ b/hermetic_build/common/setup.py @@ -10,5 +10,5 @@ python_requires=">=3.12", package_dir={ "common": ".", - } + }, ) diff --git a/hermetic_build/release_note_generation/setup.py b/hermetic_build/release_note_generation/setup.py index c7506b60d5..71728c7ef2 100755 --- a/hermetic_build/release_note_generation/setup.py +++ b/hermetic_build/release_note_generation/setup.py @@ -10,5 +10,5 @@ python_requires=">=3.12", package_dir={ "release_note_generation": ".", - } + }, ) From 3cbe3164ff2ecc1010742576c4e3a8e697958985 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 22 Oct 2024 16:01:18 -0400 Subject: [PATCH 63/69] change dockerfile --- .cloudbuild/library_generation/library_generation.Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 39f22bd0da..277853e1f9 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -80,7 +80,10 @@ RUN python -m pip install --upgrade pip # install main scripts as a python package WORKDIR / +RUN pip install --upgrade pip +RUN pip install --require-hashes -r src/common/requirements.txt RUN python -m pip install src/common +RUN pip install --require-hashes -r src/library_generation/requirements.txt RUN python -m pip install src/library_generation # Install nvm with node and npm From f7559dc20055f8b0993384f8bc6773549365a4d2 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 22 Oct 2024 16:02:48 -0400 Subject: [PATCH 64/69] change dockerfile --- .cloudbuild/library_generation/library_generation.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 277853e1f9..b1bd5c5f27 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -80,7 +80,6 @@ RUN python -m pip install --upgrade pip # install main scripts as a python package WORKDIR / -RUN pip install --upgrade pip RUN pip install --require-hashes -r src/common/requirements.txt RUN python -m pip install src/common RUN pip install --require-hashes -r src/library_generation/requirements.txt From 93ffa8a07f00463f38a29de4ba8738efbf5700a4 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 22 Oct 2024 16:03:10 -0400 Subject: [PATCH 65/69] change dockerfile --- .cloudbuild/library_generation/library_generation.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index b1bd5c5f27..1b8fee8699 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -80,9 +80,9 @@ RUN python -m pip install --upgrade pip # install main scripts as a python package WORKDIR / -RUN pip install --require-hashes -r src/common/requirements.txt +RUN python -m pip install --require-hashes -r src/common/requirements.txt RUN python -m pip install src/common -RUN pip install --require-hashes -r src/library_generation/requirements.txt +RUN python -m pip install --require-hashes -r src/library_generation/requirements.txt RUN python -m pip install src/library_generation # Install nvm with node and npm From d8d1e8e5a84357c11158ebc7d8998bce191ab940 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 22 Oct 2024 16:13:21 -0400 Subject: [PATCH 66/69] change doc --- hermetic_build/library_generation/DEVELOPMENT.md | 1 + hermetic_build/library_generation/README.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/hermetic_build/library_generation/DEVELOPMENT.md b/hermetic_build/library_generation/DEVELOPMENT.md index 51cfef7f6c..5a2375d6cb 100644 --- a/hermetic_build/library_generation/DEVELOPMENT.md +++ b/hermetic_build/library_generation/DEVELOPMENT.md @@ -21,6 +21,7 @@ code declared in a "golden branch" of the repo. It requires docker and python 3.x to be installed. ``` +python -m pip install --require-hashes -r requirements.txt python -m pip install . python -m unittest tests/integration_tests.py ``` diff --git a/hermetic_build/library_generation/README.md b/hermetic_build/library_generation/README.md index ede5a68e61..0b4208ac3e 100644 --- a/hermetic_build/library_generation/README.md +++ b/hermetic_build/library_generation/README.md @@ -193,7 +193,9 @@ The virtual environment can be installed to any folder, usually it is recommende Run the following command under the root folder of `sdk-platform-java` to install `library_generation` and its dependencies. ```bash + python -m pip install --require-hashes -r hermetic_build/common/requirements.txt python -m pip install hermetic_build/common + python -m pip install --require-hashes -r hermetic_build/library_generation/requirements.txt python -m pip install hermetic_build/library_generation ``` From 208f51ff199e2b8d798489c1621266ffd105e698 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 22 Oct 2024 19:07:10 -0400 Subject: [PATCH 67/69] do not update pip in action --- .github/scripts/action.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 6bbfdcb5a4..7c4ae22f4e 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -58,7 +58,6 @@ runs: shell: bash run: | cd "${GITHUB_WORKSPACE}" - pip install --upgrade pip pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common pip install --require-hashes -r hermetic_build/library_generation/requirements.txt From a4beff9bdde3974c3685e4764f75aaf17c031ec9 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 23 Oct 2024 10:36:46 -0400 Subject: [PATCH 68/69] change comment --- .github/workflows/verify_library_generation.yaml | 4 ++-- hermetic_build/library_generation/DEVELOPMENT.md | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index f7854a9539..c1bb381984 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -47,7 +47,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: 3.12 - - name: install python dependencies + - name: install python modules and dependencies shell: bash run: | set -ex @@ -70,7 +70,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: 3.12 - - name: install python dependencies + - name: install python modules and dependencies shell: bash run: | set -ex diff --git a/hermetic_build/library_generation/DEVELOPMENT.md b/hermetic_build/library_generation/DEVELOPMENT.md index 5a2375d6cb..df7843aeaa 100644 --- a/hermetic_build/library_generation/DEVELOPMENT.md +++ b/hermetic_build/library_generation/DEVELOPMENT.md @@ -1,5 +1,5 @@ > [!IMPORTANT] -> All examples assume you are inside the `library_generation` folder. +> All examples assume you are inside the `hermetic_build` folder. # Linting @@ -21,9 +21,9 @@ code declared in a "golden branch" of the repo. It requires docker and python 3.x to be installed. ``` -python -m pip install --require-hashes -r requirements.txt -python -m pip install . -python -m unittest tests/integration_tests.py +python -m pip install --require-hashes -r library_generation/requirements.txt +python -m pip install library_generation +python -m unittest library_generation/tests/integration_tests.py ``` # Running the unit tests @@ -34,7 +34,7 @@ Every unit test script ends with `unit_tests.py`. To avoid them specifying them individually, we can use the following command: ```bash -python -m unittest discover -s tests/ -p "*unit_tests.py" +python -m unittest discover -s library_generation/tests/ -p "*unit_tests.py" ``` > [!NOTE] @@ -132,7 +132,7 @@ python -m pip install . ### Run the script ``` -python cli/entry_point.py generate \ +python library_generation/cli/entry_point.py generate \ --repository-path="${path_to_repo}" \ --api-definitions-path="${api_definitions_path}" ``` From 80623c0f9760994a807a67c57d01a684850f4b86 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 23 Oct 2024 14:32:26 -0400 Subject: [PATCH 69/69] update requirements.txt --- hermetic_build/common/requirements.txt | 2 +- hermetic_build/library_generation/requirements.txt | 2 +- hermetic_build/release_note_generation/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hermetic_build/common/requirements.txt b/hermetic_build/common/requirements.txt index 9253e5ed5a..d952506bb5 100644 --- a/hermetic_build/common/requirements.txt +++ b/hermetic_build/common/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --generate-hashes hermetic_build/common/requirements.in diff --git a/hermetic_build/library_generation/requirements.txt b/hermetic_build/library_generation/requirements.txt index e3ed826ac7..87ac0ba921 100644 --- a/hermetic_build/library_generation/requirements.txt +++ b/hermetic_build/library_generation/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --generate-hashes hermetic_build/library_generation/requirements.in diff --git a/hermetic_build/release_note_generation/requirements.txt b/hermetic_build/release_note_generation/requirements.txt index 9eb391e0e2..5032108ff9 100644 --- a/hermetic_build/release_note_generation/requirements.txt +++ b/hermetic_build/release_note_generation/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --generate-hashes hermetic_build/release_note_generation/requirements.in