update charts
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/__main__.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/__main__.py
new file mode 100644
index 0000000..e49cc31
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/__main__.py
@@ -0,0 +1,18 @@
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from main import main
+
+if __name__ == "__main__":
+ main()
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/__init__.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/__init__.py
new file mode 100644
index 0000000..2230656
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/config/__init__.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/config/__init__.py
new file mode 100644
index 0000000..2230656
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/config/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/config/init_config.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/config/init_config.py
new file mode 100644
index 0000000..68a6f00
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/config/init_config.py
@@ -0,0 +1,90 @@
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+
+import yaml
+
+
+class InitConfig:
+ def __init__(self):
+ self.plugins = []
+ self.libs = []
+ self.plugin_cache_enabled = False
+ self.plugin_cache_dir = None
+
+ self.ca_cert_path = True
+
+ self.is_ha = False
+ self.refdb = False
+
+ def parse(self, config_file):
+ if not os.path.exists(config_file):
+ raise FileNotFoundError(f"Could not find config file: {config_file}")
+
+ with open(config_file, "r", encoding="utf-8") as f:
+ config = yaml.load(f, Loader=yaml.SafeLoader)
+
+ if config is None:
+ raise ValueError(f"Invalid config-file: {config_file}")
+
+ if "plugins" in config:
+ self.plugins = config["plugins"]
+ if "libs" in config:
+ self.libs = config["libs"]
+ # DEPRECATED: `pluginCache` was deprecated in favor of `pluginCacheEnabled`
+ if "pluginCache" in config:
+ self.plugin_cache_enabled = config["pluginCache"]
+ if "pluginCacheEnabled" in config:
+ self.plugin_cache_enabled = config["pluginCacheEnabled"]
+ if "pluginCacheDir" in config and config["pluginCacheDir"]:
+ self.plugin_cache_dir = config["pluginCacheDir"]
+
+ if "caCertPath" in config:
+ self.ca_cert_path = config["caCertPath"]
+
+ self.is_ha = "highAvailability" in config and config["highAvailability"]
+ if "refdb" in config:
+ self.refdb = config["refdb"]
+
+ return self
+
+ def get_plugins(self):
+ return self.plugins
+
+ def get_plugin_names(self):
+ return set([p["name"] for p in self.plugins])
+
+ def get_libs(self):
+ return self.libs
+
+ def get_lib_names(self):
+ return set([p["name"] for p in self.libs])
+
+ def get_packaged_plugins(self):
+ return list(filter(lambda x: "url" not in x, self.plugins))
+
+ def get_downloaded_plugins(self):
+ return list(filter(lambda x: "url" in x, self.plugins))
+
+ def get_plugins_installed_as_lib(self):
+ return [
+ lib["name"]
+ for lib in list(
+ filter(
+ lambda x: "installAsLibrary" in x and x["installAsLibrary"],
+ self.plugins,
+ )
+ )
+ ]
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/helpers/__init__.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/helpers/__init__.py
new file mode 100644
index 0000000..2230656
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/helpers/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/helpers/git.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/helpers/git.py
new file mode 100644
index 0000000..f21b28d
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/helpers/git.py
@@ -0,0 +1,76 @@
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+
+
+class GitConfigParser:
+ def __init__(self, config_path):
+ self.path = config_path
+
+ def _execute_shell_command_and_get_output_lines(self, command):
+ sub_process_run = subprocess.run(
+ command.split(), stdout=subprocess.PIPE, check=True, universal_newlines=True
+ )
+ return [line.strip() for line in sub_process_run.stdout.splitlines()]
+
+ def _get_value(self, key):
+ command = f"git config -f {self.path} --get {key}"
+ return self._execute_shell_command_and_get_output_lines(command)
+
+ def list(self):
+ command = f"git config -f {self.path} --list"
+ options = self._execute_shell_command_and_get_output_lines(command)
+ option_list = []
+ for opt in options:
+ parsed_opt = {}
+ full_key, value = opt.split("=", 1)
+ parsed_opt["value"] = value
+ full_key = full_key.split(".")
+ parsed_opt["section"] = full_key[0]
+ if len(full_key) == 2:
+ parsed_opt["subsection"] = None
+ parsed_opt["key"] = full_key[1]
+ elif len(full_key) == 3:
+ parsed_opt["subsection"] = full_key[1]
+ parsed_opt["key"] = full_key[2]
+ option_list.append(parsed_opt)
+
+ return option_list
+
+ def get(self, key, default=None):
+ """
+ Returns value of given key in the configuration file. If the key appears
+ multiple times, the last value is returned.
+ """
+ try:
+ return self._get_value(key)[-1]
+ except subprocess.CalledProcessError:
+ return default
+
+ def get_boolean(self, key, default=False):
+ """
+ Returns boolean value of given key in the configuration file. If the key
+ appears multiple times, the last value is returned.
+ """
+ if not isinstance(default, bool):
+ raise TypeError("Default has to be a boolean.")
+
+ try:
+ value = self._get_value(key)[-1].lower()
+ if value not in ["true", "false"]:
+ raise TypeError("Value is not a boolean.")
+ return value == "true"
+ except subprocess.CalledProcessError:
+ return default
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/helpers/log.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/helpers/log.py
new file mode 100644
index 0000000..06aa72c
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/helpers/log.py
@@ -0,0 +1,26 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+
+def get_logger(name):
+ log = logging.Logger(name)
+ handler = logging.StreamHandler()
+ handler.setFormatter(logging.Formatter("[%(asctime)s] %(levelname)s %(message)s"))
+ log.addHandler(handler)
+ log.setLevel(logging.DEBUG)
+ return log
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/__init__.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/__init__.py
new file mode 100644
index 0000000..2230656
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/download_plugins.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/download_plugins.py
new file mode 100755
index 0000000..2c9ace0
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/download_plugins.py
@@ -0,0 +1,372 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import os
+import shutil
+import time
+
+from abc import ABC, abstractmethod
+from zipfile import ZipFile
+
+import requests
+
+from ..helpers import log
+
+LOG = log.get_logger("init")
+MAX_LOCK_LIFETIME = 60
+MAX_CACHED_VERSIONS = 5
+
+REQUIRED_PLUGINS = ["healthcheck"]
+REQUIRED_HA_PLUGINS = ["high-availability"]
+REQUIRED_HA_LIBS = ["high-availability", "global-refdb"]
+
+
+class InvalidPluginException(Exception):
+ """Exception to be raised, if the downloaded plugin is not valid."""
+
+
+class MissingRequiredPluginException(Exception):
+ """Exception to be raised, if the downloaded plugin is not valid."""
+
+
+class AbstractPluginInstaller(ABC):
+ def __init__(self, site, config):
+ self.site = site
+ self.config = config
+
+ self.required_plugins = self._get_required_plugins()
+ self.required_libs = self._get_required_libs()
+
+ self.plugin_dir = os.path.join(site, "plugins")
+ self.lib_dir = os.path.join(site, "lib")
+ self.plugins_changed = False
+
+ def _create_plugins_dir(self):
+ if not os.path.exists(self.plugin_dir):
+ os.makedirs(self.plugin_dir)
+ LOG.info("Created plugin installation directory: %s", self.plugin_dir)
+
+ def _create_lib_dir(self):
+ if not os.path.exists(self.lib_dir):
+ os.makedirs(self.lib_dir)
+ LOG.info("Created lib installation directory: %s", self.lib_dir)
+
+ def _get_installed_plugins(self):
+ return self._get_installed_jars(self.plugin_dir)
+
+ def _get_installed_libs(self):
+ return self._get_installed_jars(self.lib_dir)
+
+ @staticmethod
+ def _get_installed_jars(dir):
+ if os.path.exists(dir):
+ return [f for f in os.listdir(dir) if f.endswith(".jar")]
+
+ return []
+
+ def _get_required_plugins(self):
+ required = REQUIRED_PLUGINS.copy()
+ if self.config.is_ha:
+ required.extend(REQUIRED_HA_PLUGINS)
+ if self.config.refdb:
+ required.append(f"{self.config.refdb}-refdb")
+ LOG.info("Requiring plugins: %s", required)
+ return required
+
+ def _get_required_libs(self):
+ required = []
+ if self.config.is_ha:
+ required.extend(REQUIRED_HA_LIBS)
+ LOG.info("Requiring libs: %s", required)
+ return required
+
+ def _install_required_plugins(self):
+ for plugin in self.required_plugins:
+ if plugin in self.config.get_plugin_names():
+ continue
+
+ self._install_required_jar(plugin, self.plugin_dir)
+
+ def _install_required_libs(self):
+ for lib in self.required_libs:
+ if lib in self.config.get_lib_names():
+ continue
+
+ self._install_required_jar(lib, self.lib_dir)
+
+ def _install_required_jar(self, jar, target_dir):
+ with ZipFile("/var/war/gerrit.war", "r") as war:
+ # Lib modules can be packaged as a plugin. However, they could
+ # currently not be installed by the init pgm tool.
+ if f"WEB-INF/plugins/{jar}.jar" in war.namelist():
+ self._install_plugin_from_war(jar, target_dir)
+ return
+ try:
+ self._install_jar_from_container(jar, target_dir)
+ except FileNotFoundError:
+ raise MissingRequiredPluginException(f"Required jar {jar} was not found.")
+
+ def _install_jar_from_container(self, plugin, target_dir):
+ source_file = os.path.join("/var/plugins", plugin + ".jar")
+ target_file = os.path.join(target_dir, plugin + ".jar")
+ LOG.info(
+ "Installing plugin %s from container to %s.",
+ plugin,
+ target_file,
+ )
+ if not os.path.exists(source_file):
+ raise FileNotFoundError(
+ "Unable to find required plugin in container: " + plugin
+ )
+ if os.path.exists(target_file) and self._get_file_sha(
+ source_file
+ ) == self._get_file_sha(target_file):
+ return
+
+ shutil.copyfile(source_file, target_file)
+ self.plugins_changed = True
+
+ def _install_plugins_from_war(self):
+ for plugin in self.config.get_packaged_plugins():
+ self._install_plugin_from_war(plugin["name"], self.plugin_dir)
+
+ def _install_plugin_from_war(self, plugin, target_dir):
+ LOG.info("Installing packaged plugin %s.", plugin)
+ with ZipFile("/var/war/gerrit.war", "r") as war:
+ war.extract(f"WEB-INF/plugins/{plugin}.jar", self.plugin_dir)
+
+ source_file = f"{self.plugin_dir}/WEB-INF/plugins/{plugin}.jar"
+ target_file = os.path.join(target_dir, f"{plugin}.jar")
+ if not os.path.exists(target_file) or self._get_file_sha(
+ source_file
+ ) != self._get_file_sha(target_file):
+ os.rename(source_file, target_file)
+ self.plugins_changed = True
+
+ shutil.rmtree(os.path.join(self.plugin_dir, "WEB-INF"), ignore_errors=True)
+
+ @staticmethod
+ def _get_file_sha(file):
+ file_hash = hashlib.sha1()
+ with open(file, "rb") as f:
+ while True:
+ chunk = f.read(64000)
+ if not chunk:
+ break
+ file_hash.update(chunk)
+
+ LOG.debug("SHA1 of file '%s' is %s", file, file_hash.hexdigest())
+
+ return file_hash.hexdigest()
+
+ def _remove_unwanted_plugins(self):
+ wanted_plugins = list(self.config.get_plugins())
+ wanted_plugins.extend(self.required_plugins)
+ self._remove_unwanted(
+ wanted_plugins, self._get_installed_plugins(), self.plugin_dir
+ )
+
+ def _remove_unwanted_libs(self):
+ wanted_libs = list(self.config.get_libs())
+ wanted_libs.extend(self.required_libs)
+ wanted_libs.extend(self.config.get_plugins_installed_as_lib())
+ self._remove_unwanted(wanted_libs, self._get_installed_libs(), self.lib_dir)
+
+ @staticmethod
+ def _remove_unwanted(wanted, installed, dir):
+ for plugin in installed:
+ if os.path.splitext(plugin)[0] not in wanted:
+ os.remove(os.path.join(dir, plugin))
+ LOG.info("Removed plugin %s", plugin)
+
+ def _symlink_plugins_to_lib(self):
+ if not os.path.exists(self.lib_dir):
+ os.makedirs(self.lib_dir)
+ else:
+ for f in os.listdir(self.lib_dir):
+ path = os.path.join(self.lib_dir, f)
+ if (
+ os.path.islink(path)
+ and os.path.splitext(f)[0]
+ not in self.config.get_plugins_installed_as_lib()
+ ):
+ os.unlink(path)
+ LOG.info("Removed symlink %s", f)
+ for lib in self.config.get_plugins_installed_as_lib():
+ plugin_path = os.path.join(self.plugin_dir, f"{lib}.jar")
+ if os.path.exists(plugin_path):
+ try:
+ os.symlink(plugin_path, os.path.join(self.lib_dir, f"{lib}.jar"))
+ except FileExistsError:
+ continue
+ else:
+ raise FileNotFoundError(
+ f"Could not find plugin {lib} to symlink to lib-directory."
+ )
+
+ def execute(self):
+ self._create_plugins_dir()
+ self._create_lib_dir()
+
+ self._remove_unwanted_plugins()
+ self._remove_unwanted_libs()
+
+ self._install_required_plugins()
+ self._install_required_libs()
+
+ self._install_plugins_from_war()
+
+ for plugin in self.config.get_downloaded_plugins():
+ self._install_plugin(plugin)
+
+ for plugin in self.config.get_libs():
+ self._install_lib(plugin)
+
+ self._symlink_plugins_to_lib()
+
+ def _download_plugin(self, plugin, target):
+ LOG.info("Downloading %s plugin to %s", plugin["name"], target)
+ try:
+ response = requests.get(plugin["url"])
+ except requests.exceptions.SSLError:
+ response = requests.get(plugin["url"], verify=self.config.ca_cert_path)
+
+ with open(target, "wb") as f:
+ f.write(response.content)
+
+ file_sha = self._get_file_sha(target)
+
+ if file_sha != plugin["sha1"]:
+ os.remove(target)
+ raise InvalidPluginException(
+ (
+ f"SHA1 of downloaded file ({file_sha}) did not match "
+ f"expected SHA1 ({plugin['sha1']}). "
+ f"Removed downloaded file ({target})"
+ )
+ )
+
+ def _install_plugin(self, plugin):
+ self._install_jar(plugin, self.plugin_dir)
+
+ def _install_lib(self, lib):
+ self._install_jar(lib, self.lib_dir)
+
+ @abstractmethod
+ def _install_jar(self, plugin, target_dir):
+ pass
+
+
+class PluginInstaller(AbstractPluginInstaller):
+ def _install_jar(self, plugin, target_dir):
+ target = os.path.join(target_dir, f"{plugin['name']}.jar")
+ if os.path.exists(target) and self._get_file_sha(target) == plugin["sha1"]:
+ return
+
+ self._download_plugin(plugin, target)
+
+ self.plugins_changed = True
+
+
+class CachedPluginInstaller(AbstractPluginInstaller):
+ @staticmethod
+ def _cleanup_cache(plugin_cache_dir):
+ cached_files = [
+ os.path.join(plugin_cache_dir, f) for f in os.listdir(plugin_cache_dir)
+ ]
+ while len(cached_files) > MAX_CACHED_VERSIONS:
+ oldest_file = min(cached_files, key=os.path.getctime)
+ LOG.info(
+ "Too many cached files in %s. Removing file %s",
+ plugin_cache_dir,
+ oldest_file,
+ )
+ os.remove(oldest_file)
+ cached_files.remove(oldest_file)
+
+ @staticmethod
+ def _create_download_lock(lock_path):
+ with open(lock_path, "w", encoding="utf-8") as f:
+ f.write(os.environ["HOSTNAME"])
+ LOG.debug("Created download lock %s", lock_path)
+
+ @staticmethod
+ def _create_plugin_cache_dir(plugin_cache_dir):
+ if not os.path.exists(plugin_cache_dir):
+ os.makedirs(plugin_cache_dir)
+ LOG.info("Created cache directory %s", plugin_cache_dir)
+
+ def _get_cached_plugin_path(self, plugin):
+ return os.path.join(
+ self.config.plugin_cache_dir,
+ plugin["name"],
+ f"{plugin['name']}-{plugin['sha1']}.jar",
+ )
+
+ def _install_from_cache_or_download(self, plugin, target):
+ cached_plugin_path = self._get_cached_plugin_path(plugin)
+
+ if os.path.exists(cached_plugin_path):
+ LOG.info("Installing %s plugin from cache.", plugin["name"])
+ else:
+ LOG.info("%s not found in cache. Downloading it.", plugin["name"])
+ self._create_plugin_cache_dir(os.path.dirname(cached_plugin_path))
+
+ lock_path = f"{cached_plugin_path}.lock"
+ while os.path.exists(lock_path):
+ LOG.info(
+ "Download lock found (%s). Waiting %d seconds for it to be released.",
+ lock_path,
+ MAX_LOCK_LIFETIME,
+ )
+ lock_timestamp = os.path.getmtime(lock_path)
+ if time.time() > lock_timestamp + MAX_LOCK_LIFETIME:
+ LOG.info("Stale download lock found (%s).", lock_path)
+ self._remove_download_lock(lock_path)
+
+ self._create_download_lock(lock_path)
+
+ try:
+ self._download_plugin(plugin, cached_plugin_path)
+ finally:
+ self._remove_download_lock(lock_path)
+
+ shutil.copy(cached_plugin_path, target)
+ self._cleanup_cache(os.path.dirname(cached_plugin_path))
+
+ def _install_jar(self, plugin, target_dir):
+ install_path = os.path.join(target_dir, f"{plugin['name']}.jar")
+ if (
+ os.path.exists(install_path)
+ and self._get_file_sha(install_path) == plugin["sha1"]
+ ):
+ return
+
+ self.plugins_changed = True
+ self._install_from_cache_or_download(plugin, install_path)
+
+ @staticmethod
+ def _remove_download_lock(lock_path):
+ os.remove(lock_path)
+ LOG.debug("Removed download lock %s", lock_path)
+
+
+def get_installer(site, config):
+ plugin_installer = (
+ CachedPluginInstaller if config.plugin_cache_enabled else PluginInstaller
+ )
+ return plugin_installer(site, config)
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/init.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/init.py
new file mode 100755
index 0000000..4931984
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/init.py
@@ -0,0 +1,227 @@
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import subprocess
+import sys
+
+from ..helpers import git, log
+from .download_plugins import get_installer
+from .reindex import IndexType, get_reindexer
+from .validate_notedb import NoteDbValidator
+
+LOG = log.get_logger("init")
+MNT_PATH = "/var/mnt"
+
+
+class GerritInit:
+ def __init__(self, site, config):
+ self.site = site
+ self.config = config
+
+ self.plugin_installer = get_installer(self.site, self.config)
+
+ self.gerrit_config = git.GitConfigParser(
+ os.path.join(MNT_PATH, "etc/config/gerrit.config")
+ )
+ self.is_online_reindex = self.gerrit_config.get_boolean(
+ "index.onlineUpgrade", True
+ )
+ self.force_offline_reindex = False
+ self.installed_plugins = self._get_installed_plugins()
+
+ self.is_replica = self.gerrit_config.get_boolean("container.replica")
+ self.pid_file = f"{self.site}/logs/gerrit.pid"
+
+ def _get_gerrit_version(self, gerrit_war_path):
+ command = f"java -jar {gerrit_war_path} version"
+ version_process = subprocess.run(
+ command.split(), stdout=subprocess.PIPE, check=True
+ )
+ return version_process.stdout.decode().strip()
+
+ def _get_installed_plugins(self):
+ plugin_path = os.path.join(self.site, "plugins")
+ installed_plugins = set()
+
+ if os.path.exists(plugin_path):
+ for f in os.listdir(plugin_path):
+ if os.path.isfile(os.path.join(plugin_path, f)) and f.endswith(".jar"):
+ installed_plugins.add(os.path.splitext(f)[0])
+
+ return installed_plugins
+
+ def _gerrit_war_updated(self):
+ installed_war_path = os.path.join(self.site, "bin", "gerrit.war")
+ installed_version = self._get_gerrit_version(installed_war_path)
+ provided_version = self._get_gerrit_version("/var/war/gerrit.war")
+ LOG.info(
+ "Installed Gerrit version: %s; Provided Gerrit version: %s). ",
+ installed_version,
+ provided_version,
+ )
+ installed_minor_version = installed_version.split(".")[0:2]
+ provided_minor_version = provided_version.split(".")[0:2]
+
+ if (
+ not self.is_online_reindex
+ and installed_minor_version != provided_minor_version
+ ):
+ self.force_offline_reindex = True
+ return installed_version != provided_version
+
+ def _needs_init(self):
+ installed_war_path = os.path.join(self.site, "bin", "gerrit.war")
+ if not os.path.exists(installed_war_path):
+ LOG.info("Gerrit is not yet installed. Initializing new site.")
+ return True
+
+ if self._gerrit_war_updated():
+ LOG.info("Reinitializing site to perform update.")
+ return True
+
+ if self.plugin_installer.plugins_changed:
+ LOG.info("Plugins were installed or updated. Initializing.")
+ return True
+
+ if self.config.get_plugin_names().difference(self.installed_plugins):
+ LOG.info("Reininitializing site to install additional plugins.")
+ return True
+
+ LOG.info("No initialization required.")
+ return False
+
+ def _ensure_symlink(self, src, target):
+ if not os.path.exists(src):
+ raise FileNotFoundError(f"Unable to find mounted dir: {src}")
+
+ if os.path.islink(target) and os.path.realpath(target) == src:
+ return
+
+ if os.path.exists(target):
+ if os.path.isdir(target) and not os.path.islink(target):
+ shutil.rmtree(target)
+ else:
+ os.remove(target)
+
+ os.symlink(src, target)
+
+ def _symlink_mounted_site_components(self):
+ self._ensure_symlink(f"{MNT_PATH}/git", f"{self.site}/git")
+ self._ensure_symlink(f"{MNT_PATH}/logs", f"{self.site}/logs")
+
+ mounted_shared_dir = f"{MNT_PATH}/shared"
+ if not self.is_replica and os.path.exists(mounted_shared_dir):
+ self._ensure_symlink(mounted_shared_dir, f"{self.site}/shared")
+
+ index_type = self.gerrit_config.get("index.type", default=IndexType.LUCENE.name)
+ if IndexType[index_type.upper()] is IndexType.ELASTICSEARCH:
+ self._ensure_symlink(f"{MNT_PATH}/index", f"{self.site}/index")
+
+ data_dir = f"{self.site}/data"
+ if os.path.exists(data_dir):
+ for file_or_dir in os.listdir(data_dir):
+ abs_path = os.path.join(data_dir, file_or_dir)
+ if os.path.islink(abs_path) and not os.path.exists(
+ os.path.realpath(abs_path)
+ ):
+ os.unlink(abs_path)
+ else:
+ os.makedirs(data_dir)
+
+ mounted_data_dir = f"{MNT_PATH}/data"
+ if os.path.exists(mounted_data_dir):
+ for file_or_dir in os.listdir(mounted_data_dir):
+ abs_path = os.path.join(data_dir, file_or_dir)
+ abs_mounted_path = os.path.join(mounted_data_dir, file_or_dir)
+ if os.path.isdir(abs_mounted_path):
+ self._ensure_symlink(abs_mounted_path, abs_path)
+
+ def _symlink_configuration(self):
+ etc_dir = f"{self.site}/etc"
+ if not os.path.exists(etc_dir):
+ os.makedirs(etc_dir)
+
+ for config_type in ["config", "secret"]:
+ if os.path.exists(f"{MNT_PATH}/etc/{config_type}"):
+ for file_or_dir in os.listdir(f"{MNT_PATH}/etc/{config_type}"):
+ if os.path.isfile(
+ os.path.join(f"{MNT_PATH}/etc/{config_type}", file_or_dir)
+ ):
+ self._ensure_symlink(
+ os.path.join(f"{MNT_PATH}/etc/{config_type}", file_or_dir),
+ os.path.join(etc_dir, file_or_dir),
+ )
+
+ def _remove_auto_generated_ssh_keys(self):
+ etc_dir = f"{self.site}/etc"
+ if not os.path.exists(etc_dir):
+ return
+
+ for file_or_dir in os.listdir(etc_dir):
+ full_path = os.path.join(etc_dir, file_or_dir)
+ if os.path.isfile(full_path) and file_or_dir.startswith("ssh_host_"):
+ os.remove(full_path)
+
+ def execute(self):
+ if not self.is_replica:
+ self._symlink_mounted_site_components()
+ elif not NoteDbValidator(MNT_PATH).check():
+ LOG.info("NoteDB not ready. Initializing repositories.")
+ self._symlink_mounted_site_components()
+ self._symlink_configuration()
+
+ if os.path.exists(self.pid_file):
+ os.remove(self.pid_file)
+
+ self.plugin_installer.execute()
+
+ if self._needs_init():
+ if self.gerrit_config:
+ LOG.info("Existing gerrit.config found.")
+ dev_option = (
+ "--dev"
+ if self.gerrit_config.get(
+ "auth.type", "development_become_any_account"
+ ).lower()
+ == "development_become_any_account"
+ else ""
+ )
+ else:
+ LOG.info("No gerrit.config found. Initializing default site.")
+ dev_option = "--dev"
+
+ flags = f"--no-auto-start --batch {dev_option}"
+
+ command = f"java -jar /var/war/gerrit.war init {flags} -d {self.site}"
+
+ init_process = subprocess.run(
+ command.split(), stdout=subprocess.PIPE, check=True
+ )
+
+ if init_process.returncode > 0:
+ LOG.error(
+ "An error occurred, when initializing Gerrit. Exit code: %d",
+ init_process.returncode,
+ )
+ sys.exit(1)
+
+ self._remove_auto_generated_ssh_keys()
+ self._symlink_configuration()
+
+ if self.is_replica:
+ self._symlink_mounted_site_components()
+
+ get_reindexer(self.site, self.config).start(self.force_offline_reindex)
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/reindex.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/reindex.py
new file mode 100755
index 0000000..e5ec6df
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/reindex.py
@@ -0,0 +1,208 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+import enum
+import os.path
+import subprocess
+import sys
+
+import requests
+
+from ..helpers import git, log
+
+LOG = log.get_logger("reindex")
+MNT_PATH = "/var/mnt"
+INDEXES_PRIMARY = set(["accounts", "changes", "groups", "projects"])
+INDEXES_REPLICA = set(["groups"])
+
+
+class IndexType(enum.Enum):
+ LUCENE = enum.auto()
+ ELASTICSEARCH = enum.auto()
+
+
+class GerritAbstractReindexer(abc.ABC):
+ def __init__(self, gerrit_site_path, config):
+ self.gerrit_site_path = gerrit_site_path
+ self.index_config_path = f"{self.gerrit_site_path}/index/gerrit_index.config"
+ self.init_config = config
+
+ self.gerrit_config = git.GitConfigParser(
+ os.path.join(MNT_PATH, "etc/config/gerrit.config")
+ )
+ self.is_online_reindex = self.gerrit_config.get_boolean(
+ "index.onlineUpgrade", True
+ )
+ self.is_replica = self.gerrit_config.get_boolean("container.replica", False)
+
+ self.configured_indices = self._parse_gerrit_index_config()
+
+ @abc.abstractmethod
+ def _get_indices(self):
+ pass
+
+ def _parse_gerrit_index_config(self):
+ indices = {}
+ if os.path.exists(self.index_config_path):
+ config = git.GitConfigParser(self.index_config_path)
+ options = config.list()
+ for opt in options:
+ name, version = opt["subsection"].rsplit("_", 1)
+ ready = opt["value"].lower() == "true"
+ if name in indices:
+ indices[name] = {
+ "read": version if ready else indices[name]["read"],
+ "latest_write": max(version, indices[name]["latest_write"]),
+ }
+ else:
+ indices[name] = {
+ "read": version if ready else None,
+ "latest_write": version,
+ }
+ return indices
+
+ def _get_not_ready_indices(self):
+ not_ready_indices = []
+ for index, index_attrs in self.configured_indices.items():
+ if not index_attrs["read"]:
+ LOG.info("Index %s not ready.", index)
+ not_ready_indices.append(index)
+ index_set = INDEXES_REPLICA if self.is_replica else INDEXES_PRIMARY
+ not_ready_indices.extend(index_set.difference(self.configured_indices.keys()))
+ return not_ready_indices
+
+ def _indexes_need_update(self):
+ indices = self._get_indices()
+
+ if not indices:
+ return True
+
+ for index, index_attrs in self.configured_indices.items():
+ if (
+ index not in indices
+ or index_attrs["latest_write"] != indices[index]
+ or index_attrs["read"] != index_attrs["latest_write"]
+ ):
+ return True
+ return False
+
+ def reindex(self, indices=None):
+ LOG.info("Starting to reindex.")
+ command = f"java -jar /var/war/gerrit.war reindex -d {self.gerrit_site_path}"
+
+ if indices:
+ command += " ".join([f" --index {i}" for i in indices])
+
+ reindex_process = subprocess.run(
+ command.split(), stdout=subprocess.PIPE, check=True
+ )
+
+ if reindex_process.returncode > 0:
+ LOG.error(
+ "An error occurred, when reindexing Gerrit indices. Exit code: %d",
+ reindex_process.returncode,
+ )
+ sys.exit(1)
+
+ LOG.info("Finished reindexing.")
+
+ def start(self, is_forced):
+ if is_forced:
+ self.reindex()
+ return
+
+ if not self.configured_indices:
+ LOG.info("gerrit_index.config does not exist. Creating all indices.")
+ self.reindex()
+ return
+
+ not_ready_indices = self._get_not_ready_indices()
+ if not_ready_indices:
+ self.reindex(not_ready_indices)
+
+ if not self.is_online_reindex and self._indexes_need_update():
+ LOG.info("Not all indices are up-to-date.")
+ self.reindex()
+ return
+
+ LOG.info("Skipping reindexing.")
+
+
+class GerritLuceneReindexer(GerritAbstractReindexer):
+ def _get_indices(self):
+ file_list = os.listdir(os.path.join(self.gerrit_site_path, "index"))
+ file_list.remove("gerrit_index.config")
+ lucene_indices = {}
+ for index in file_list:
+ try:
+ (name, version) = index.split("_")
+ if name in lucene_indices:
+ lucene_indices[name] = max(version, lucene_indices[name])
+ else:
+ lucene_indices[name] = version
+ except ValueError:
+ LOG.debug("Ignoring invalid file in index-directory: %s", index)
+ return lucene_indices
+
+
+class GerritElasticSearchReindexer(GerritAbstractReindexer):
+ def _get_elasticsearch_config(self):
+ es_config = {}
+ gerrit_config = git.GitConfigParser(
+ os.path.join(self.gerrit_site_path, "etc", "gerrit.config")
+ )
+ es_config["prefix"] = gerrit_config.get(
+ "elasticsearch.prefix", default=""
+ ).lower()
+ es_config["server"] = gerrit_config.get(
+ "elasticsearch.server", default=""
+ ).lower()
+ return es_config
+
+ def _get_indices(self):
+ es_config = self._get_elasticsearch_config()
+ url = f"{es_config['server']}/{es_config['prefix']}*"
+ try:
+ response = requests.get(url)
+ except requests.exceptions.SSLError:
+ response = requests.get(url, verify=self.init_config.ca_cert_path)
+
+ es_indices = {}
+ for index, _ in response.json().items():
+ try:
+ index = index.replace(es_config["prefix"], "", 1)
+ (name, version) = index.split("_")
+ es_indices[name] = version
+ except ValueError:
+ LOG.debug("Found unknown index: %s", index)
+
+ return es_indices
+
+
+def get_reindexer(gerrit_site_path, config):
+ gerrit_config = git.GitConfigParser(
+ os.path.join(gerrit_site_path, "etc", "gerrit.config")
+ )
+ index_type = gerrit_config.get("index.type", default=IndexType.LUCENE.name)
+
+ if IndexType[index_type.upper()] is IndexType.LUCENE:
+ return GerritLuceneReindexer(gerrit_site_path, config)
+
+ if IndexType[index_type.upper()] is IndexType.ELASTICSEARCH:
+ return GerritElasticSearchReindexer(gerrit_site_path, config)
+
+ raise RuntimeError(f"Unknown index type {index_type}.")
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/validate_notedb.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/validate_notedb.py
new file mode 100644
index 0000000..aff9ce6
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/initializer/tasks/validate_notedb.py
@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import time
+
+from ..helpers import log
+
+LOG = log.get_logger("init")
+
+
+class NoteDbValidator:
+ def __init__(self, site):
+ self.site = site
+
+ self.notedb_repos = ["All-Projects.git", "All-Users.git"]
+ self.required_refs = {
+ "All-Projects.git": ["refs/meta/config", "refs/meta/version"],
+ "All-Users.git": ["refs/meta/config"],
+ }
+
+ def _test_repo_exists(self, repo):
+ return os.path.exists(os.path.join(self.site, "git", repo))
+
+ def _test_ref_exists(self, repo, ref):
+ command = f"git --git-dir {self.site}/git/{repo} rev-parse --verify {ref}"
+ git_show_ref = subprocess.run(
+ command.split(),
+ stdout=subprocess.PIPE,
+ universal_newlines=True,
+ check=False,
+ )
+
+ return git_show_ref.returncode == 0
+
+ def wait_until_valid(self):
+ for repo in self.notedb_repos:
+ LOG.info("Waiting for repository %s.", repo)
+ while not self._test_repo_exists(repo):
+ time.sleep(1)
+ LOG.info("Found %s.", repo)
+
+ for ref in self.required_refs[repo]:
+ LOG.info("Waiting for ref %s in repository %s.", ref, repo)
+ while not self._test_ref_exists(repo, ref):
+ time.sleep(1)
+ LOG.info("Found ref %s in repo %s.", ref, repo)
+
+ def check(self):
+ for repo in self.notedb_repos:
+ if not self._test_repo_exists(repo):
+ LOG.info("Repository %s is missing.", repo)
+ return False
+ LOG.info("Found %s.", repo)
+
+ for ref in self.required_refs[repo]:
+ if not self._test_ref_exists(repo, ref):
+ LOG.info("Ref %s in repository %s is missing.", ref, repo)
+ return False
+ LOG.info("Found ref %s in repo %s.", ref, repo)
+ return True
diff --git a/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/main.py b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/main.py
new file mode 100755
index 0000000..b41cf3a
--- /dev/null
+++ b/charts/k8s-gerrit/container-images/gerrit-init/tools/gerrit-initializer/main.py
@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+
+from initializer.tasks import download_plugins, init, reindex, validate_notedb
+from initializer.config.init_config import InitConfig
+
+
+def _run_download_plugins(args):
+ config = InitConfig().parse(args.config)
+ download_plugins.get_installer(args.site, config).execute()
+
+
+def _run_init(args):
+ config = InitConfig().parse(args.config)
+ init.GerritInit(args.site, config).execute()
+
+
+def _run_reindex(args):
+ config = InitConfig().parse(args.config)
+ reindex.get_reindexer(args.site, config).start(args.force)
+
+
+def _run_validate_notedb(args):
+ validate_notedb.NoteDbValidator(args.site).wait_until_valid()
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "-s",
+ "--site",
+ help="Path to Gerrit site",
+ dest="site",
+ action="store",
+ default="/var/gerrit",
+ required=True,
+ )
+ parser.add_argument(
+ "-c",
+ "--config",
+ help="Path to configuration file for init process.",
+ dest="config",
+ action="store",
+ required=True,
+ )
+
+ subparsers = parser.add_subparsers()
+
+ parser_download_plugins = subparsers.add_parser(
+ "download-plugins", help="Download plugins"
+ )
+ parser_download_plugins.set_defaults(func=_run_download_plugins)
+
+ parser_init = subparsers.add_parser("init", help="Initialize Gerrit site")
+ parser_init.set_defaults(func=_run_init)
+
+ parser_reindex = subparsers.add_parser("reindex", help="Reindex Gerrit indexes")
+ parser_reindex.add_argument(
+ "-f",
+ "--force",
+ help="Reindex even if indices are ready.",
+ dest="force",
+ action="store_true",
+ )
+ parser_reindex.set_defaults(func=_run_reindex)
+
+ parser_validate_notedb = subparsers.add_parser(
+ "validate-notedb", help="Validate NoteDB"
+ )
+ parser_validate_notedb.set_defaults(func=_run_validate_notedb)
+
+ args = parser.parse_args()
+ args.func(args)
+
+
+if __name__ == "__main__":
+ main()