diff --git a/.aws/buckets.sh b/.aws/buckets.sh new file mode 100644 index 0000000..d87d333 --- /dev/null +++ b/.aws/buckets.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -x +# awslocal s3 mb s3://test-static.quizzop.com +aws --endpoint-url=http://127.0.0.1:4566 s3api create-bucket --bucket dev-discovery --create-bucket-configuration LocationConstraint=ap-south-1 +set +x \ No newline at end of file diff --git a/.gitignore b/.gitignore index 45baf38..958a16a 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,9 @@ npm-debug.log /minikube/ -/.vscode/ \ No newline at end of file +/.vscode + +/.env + +example_workflow.sh +test_gitops.exs \ No newline at end of file diff --git a/.tool-versions b/.tool-versions index 29940d1..7a6e4d6 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,3 @@ -elixir 1.12.2-otp-24 -erlang 24.0 +elixir 1.18.2-otp-27 +erlang 27.2 +nodejs 19.8.1 diff --git a/assets/.tool-versions b/assets/.tool-versions new file mode 100644 index 0000000..60c6d23 --- /dev/null +++ b/assets/.tool-versions @@ -0,0 +1 @@ +nodejs 19.8.1 diff --git a/config/config.exs b/config/config.exs index f7dc5a9..5dbffbe 100644 --- a/config/config.exs +++ b/config/config.exs @@ -20,6 +20,16 @@ config :logger, :console, format: "$time $metadata[$level] $message\n", metadata: [:request_id] +# Aws s3 config +config :ex_aws, + json_codec: Jason + +config :ex_aws, :s3, + scheme: "https://", + region: "ap-south-1", + host: "s3-ap-south-1.amazonaws.com", + port: 443 + # Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason @@ -44,15 +54,19 @@ config :discovery, service_account: "discovery-sa", use_external_ingress_class: true, ingress_class: "nginx-external", - image_pull_secrets: "dockerhub-auth-discovery" + image_pull_secrets: "dockerhub-auth-discovery", + kubernetes_arch: "amd64" config :discovery, :api_version, config_map: "v1", deployment: "apps/v1", - ingress: "networking.k8s.io/v1beta1", + ingress: "networking.k8s.io/v1", namespace: "v1", service: "v1" +config :discovery, + git_username: "ghostdsb" + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/dev.exs b/config/dev.exs index 310ddc2..6c580f6 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -12,13 +12,13 @@ config :discovery, DiscoveryWeb.Endpoint, code_reloader: true, check_origin: false, watchers: [ - node: [ - "node_modules/webpack/bin/webpack.js", - "--mode", - "development", - "--watch-stdin", - cd: Path.expand("../assets", __DIR__) - ] + # node: [ + # "node_modules/webpack/bin/webpack.js", + # "--mode", + # "development", + # "--watch-stdin", + # cd: Path.expand("../assets", __DIR__) + # ] ] # ## SSL Support @@ -56,6 +56,17 @@ config :discovery, DiscoveryWeb.Endpoint, ] ] +# uncomment for testing on localhost for quizzop question upload module +config :ex_aws, :s3, + scheme: "http://", + region: "ap-south-1", + host: "localhost", + port: 4566 + +config :discovery, + discovery_bucket: "dev-discovery", + discovery_bucket_url: "https://dev-discovery.gamezop.com" + # Do not include metadata nor timestamps in development logs config :logger, :console, format: "[$level] $message\n" @@ -71,7 +82,8 @@ config :discovery, :base_url, "http://localhost:4000" config :discovery, connection_method: :kube_config, namespace: "discovery", - service_account: "discovery-service-account", + ## "discovery-service-account", + service_account: "default", resources: %{ limits: %{cpu: "500m", memory: "500Mi"}, requests: %{cpu: "100m", memory: "300Mi"} @@ -79,3 +91,10 @@ config :discovery, image_pull_secrets: "ghostdsb-auth-discovery", use_external_ingress_class: false, use_service_account: true + +# config :discovery, :api_version, +# config_map: "v1", +# deployment: "apps/v1", +# ingress: "networking.k8s.io/v1beta1", +# namespace: "v1", +# service: "v1" diff --git a/config/runtime.exs b/config/runtime.exs index 1418de4..049154f 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -28,4 +28,18 @@ if config_env() == :prod or config_env() == :develop do # # Then you can assemble a release by calling `mix release`. # See `mix help release` for more information. + aws_access_id = System.fetch_env!("AWS_ACCESS_KEY_ID") + aws_access_key = System.fetch_env!("AWS_SECRET_ACCESS_KEY") + discovery_bucket = System.fetch_env!("DISCOVERY_BUCKET") + discovery_bucket_url = System.fetch_env!("DISCOVERY_BUCKET_URL") + git_access_token = System.fetch_env!("GITHUB_REPO_TOKEN") + + config :ex_aws, + access_key_id: aws_access_id, + secret_access_key: aws_access_key + + config :discovery, + discovery_bucket: discovery_bucket, + discovery_bucket_url: discovery_bucket_url, + git_access_token: git_access_token end diff --git a/config/test.exs b/config/test.exs index 5328d9e..8a03589 100644 --- a/config/test.exs +++ b/config/test.exs @@ -8,3 +8,17 @@ config :discovery, DiscoveryWeb.Endpoint, # Print only warnings and errors during test config :logger, level: :warn + +config :ex_aws, :s3, + scheme: "http://", + region: "ap-south-1", + host: "localhost", + port: 4566 + +config :ex_aws, + access_key_id: "secret", + secret_access_key: "secret" + +config :discovery, + discovery_bucket: "discovery_bucket", + discovery_bucket_url: "discovery_bucket_url" diff --git a/dev.Dockerfile b/dev.Dockerfile index 94aeeac..cc3828b 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -20,8 +20,8 @@ ENV MIX_ENV="${MIX_ENV}" # install mix dependencies COPY mix.exs mix.lock ./ -RUN mix deps.get --only $MIX_ENV -RUN mkdir config +COPY config config +RUN mix do deps.get, deps.compile # copy compile-time config files before we compile dependencies # to ensure any relevant config change will trigger the dependencies diff --git a/dev.env b/dev.env index 669d67b..2efa552 100644 --- a/dev.env +++ b/dev.env @@ -1,2 +1,7 @@ -KUBERNETES_SERVICE_HOST="192.168.49.2" -KUBERNETES_SERVICE_PORT=8443 \ No newline at end of file +export KUBERNETES_SERVICE_HOST="192.168.49.2" +export KUBERNETES_SERVICE_PORT=8443 +export AWS_ACCESS_KEY_ID=key +export AWS_SECRET_ACCESS_KEY=secret +export DISCOVERY_BUCKET=dev-discovery +export DISCOVERY_BUCKET_URL=https://dev-discovery.gamezop.com +export GITHUB_REPO_TOKEN=secret \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..9e52e12 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,20 @@ +version: "3.9" + +services: + localstack: + image: localstack/localstack + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range + - "127.0.0.1:53:53" # DNS config (only required for Pro) + - "127.0.0.1:53:53/udp" # DNS config (only required for Pro) + - "127.0.0.1:443:443" # LocalStack HTTPS Gateway (only required for Pro) + environment: + - SERVICES=s3 + - DEBUG=${DEBUG-} + - DOCKER_HOST=unix:///var/run/docker.sock + - DEFAULT_REGION=ap-south-1 + - AWS_ACCESS_KEY_ID=key + - AWS_SECRET_ACCESS_KEY=secret + volumes: + - ./.aws:/docker-entrypoint-initaws.d \ No newline at end of file diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile new file mode 100644 index 0000000..a6fa13a --- /dev/null +++ b/dockerfiles/Dockerfile @@ -0,0 +1,91 @@ +FROM hexpm/elixir:1.14.0-erlang-25.1-alpine-3.15.6 as build + +# install build dependencies +RUN apk add --no-cache build-base git npm python3 curl + + +# prepare build dir +WORKDIR /app + +# install hex + rebar +RUN mix local.hex --force && \ + mix local.rebar --force + +# set build ENV +ARG MIX_ENV +ENV MIX_ENV="${MIX_ENV}" + +# install mix dependencies +COPY mix.exs mix.lock ./ +RUN --mount=type=secret,id=github_token \ + GITHUB_TOKEN=$(cat /run/secrets/github_token) && \ + echo "machine github.com login $GITHUB_TOKEN" > ~/.netrc && \ + mix deps.get --only ${MIX_ENV} +RUN mkdir config + +# copy compile-time config files before we compile dependencies +# to ensure any relevant config change will trigger the dependencies +# to be re-compiled. +COPY config/config.exs config/$MIX_ENV.exs config/ +RUN mix deps.compile + +COPY priv priv +# COPY assets assets + +# compile and build release +COPY lib lib + +RUN mix compile +# changes to config/runtime.exs don't require recompiling the code +COPY config/runtime.exs config/ + +# uncomment COPY if rel/ exists +# COPY rel rel +RUN MIX_ENV=${MIX_ENV} mix release + +# prepare release image +FROM alpine:3.15.6 AS app + +RUN apk add --no-cache openssl ncurses-libs libgcc libstdc++ git npm python3 curl + +ARG MIX_ENV +ENV USER="elixir" + +WORKDIR "/home/${USER}/app" + +# Creates an unprivileged user to be used exclusively to run the Phoenix app +# RUN \ + # addgroup \ + # -g 1000 \ + # -S "${USER}" \ + # && adduser \ + # -s /bin/sh \ + # -u 1000 \ + # -G "${USER}" \ + # -h "/home/${USER}" \ + # -D "${USER}" \ + # && su "${USER}" + +# Creates super privileged user + +RUN chown root:root /home/elixir/app + +# Everything from this line onwards will run in the context of the unprivileged user. +# USER "${USER}" + +# Everything from this line onwards will run in the context of the super privileged user. + +USER root:root + +COPY --from=build --chown=root:root /app/_build/"${MIX_ENV}"/rel/discovery ./ + +COPY ./dockerfiles/entrypoint.sh . + +ARG MIX_ENV +ENV MIX_ENV="${MIX_ENV}" + +# ARG POD_A_RECORD +# ENV POD_A_RECORD="${echo ${POD_IP} | sed 's/\\./-/g'}" + +# Run the Phoenix app +CMD ["sh", "./entrypoint.sh"] \ No newline at end of file diff --git a/dockerfiles/entrypoint.sh b/dockerfiles/entrypoint.sh new file mode 100644 index 0000000..acc34f9 --- /dev/null +++ b/dockerfiles/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +bin="bin/discovery" + +# start the elixir application +exec "$bin" "start" \ No newline at end of file diff --git a/lib/discovery/application.ex b/lib/discovery/application.ex index b4356e9..7fd607e 100644 --- a/lib/discovery/application.ex +++ b/lib/discovery/application.ex @@ -8,12 +8,15 @@ defmodule Discovery.Application do alias Discovery.Controller.DeploymentController alias Discovery.Deploy.DeployManager alias Discovery.Engine.Builder + alias Discovery.GitOps.GitOpsManager alias Discovery.Scheduler alias Discovery.Utils require Logger def start(_type, _args) do + git_access_token = Application.get_env(:discovery, :git_access_token) + children = [ # Start the Telemetry supervisor DiscoveryWeb.Telemetry, @@ -24,6 +27,23 @@ defmodule Discovery.Application do {Builder, []}, {DeploymentController, []}, {DeployManager, []}, + {GitOpsManager, + [ + repo_url: "git@github.com:gamezop/discovery-k8s.git", + token: git_access_token, + local_path: "/tmp/discovery-k8s", + use_pr: false, + write_layout: :env_first, + env_root_map: %{"dev" => "dev", "staging" => "staging", "prod" => "prod"}, + base_dir_name: "base", + file_names: %{ + deployment: "deploy.yml", + configmap: "configmap.yml", + secret: "secret.yml", + service: "service.yml", + ingress: "ingress.yml" + } + ]}, Scheduler # Start a worker by calling: Discovery.Worker.start_link(arg) # {Discovery.Worker, arg} diff --git a/lib/discovery/deploy/deploy_utils.ex b/lib/discovery/deploy/deploy_utils.ex index 5a08829..b2a95a5 100644 --- a/lib/discovery/deploy/deploy_utils.ex +++ b/lib/discovery/deploy/deploy_utils.ex @@ -103,6 +103,7 @@ defmodule Discovery.Deploy.DeployUtils do @spec delete_app(binary) :: {:ok, [binary]} | {:error, atom, binary} def delete_app(app_name) do conn = Builder.get_conn() + bucket = Application.get_env(:discovery, :discovery_bucket) Ingress.get_ingress_services(conn, app_name) |> Enum.each(fn app -> @@ -114,17 +115,31 @@ defmodule Discovery.Deploy.DeployUtils do Ingress.delete_operation(app_name) |> delete_resource(app_name) - File.rm_rf("minikube/discovery/#{app_name}") + app_location = "minikube/discovery/#{app_name}" + + File.rm_rf(app_location) + |> then(fn _ -> Discovery.S3Uploader.delete_content(bucket, app_location) end) end @spec create_namespace_directory :: :ok def create_namespace_directory do - if File.exists?("minikube/discovery/namespace.yml") do + namespace_file_location = "minikube/discovery/namespace.yml" + + if File.exists?(namespace_file_location) do Utils.puts_warn("NAMESPACE DIRECTORY EXISTS") else - File.mkdir_p!(Path.dirname("minikube/discovery/namespace.yml")) + File.mkdir_p!(Path.dirname(namespace_file_location)) namespace_template = File.read!("#{:code.priv_dir(:discovery)}/templates/namespace.yml") - File.write!("minikube/discovery/namespace.yml", namespace_template) + File.write!(namespace_file_location, namespace_template) + + bucket = Application.get_env(:discovery, :discovery_bucket) + + # uploads namespace to S3 + Discovery.S3Uploader.upload_file(namespace_file_location, bucket, namespace_file_location) + + # downloads all files from S3, its ok if we rewrite namespace, by this we get the config files already + # uploaded to S3 by previous deployment + Discovery.S3Uploader.download_contents(bucket) Utils.puts_warn("RUNNING NAMESPACE: discovery") end end @@ -138,16 +153,20 @@ defmodule Discovery.Deploy.DeployUtils do end end - @spec create_app_folder(app()) :: :ok | {:error, term()} + @spec create_app_version_folder(app()) :: :ok | {:error, term()} defp create_app_version_folder(app) do File.mkdir("minikube/discovery/#{app.app_name}/#{app.app_name}-#{app.uid}") end @spec delete_app_version_folder(del_deployment()) :: {:ok, list()} | {:error, String.t()} defp delete_app_version_folder(app) do - File.rm_rf("minikube/discovery/#{app.app_name}/#{app.app_name}-#{app.uid}") + app_version_location = "minikube/discovery/#{app.app_name}/#{app.app_name}-#{app.uid}" + bucket = Application.get_env(:discovery, :discovery_bucket) + + File.rm_rf(app_version_location) |> case do {:ok, list} -> + Discovery.S3Uploader.delete_content(bucket, app_version_location) {:ok, list} {:error, reason, _} -> @@ -264,7 +283,7 @@ defmodule Discovery.Deploy.DeployUtils do @spec patch_resource(String.t()) :: :ok | {:error, String.t()} defp patch_resource(resource) do - Utils.puts_warn("RUNNING RESOURCE: #{resource}") + Utils.puts_warn("PATCHING RESOURCE: #{resource}") with conn when not is_nil(conn) <- Builder.get_conn(), {:ok, resource_map} <- K8s.Resource.from_file(resource), diff --git a/lib/discovery/gitops/config_fetcher.ex b/lib/discovery/gitops/config_fetcher.ex new file mode 100644 index 0000000..24974f3 --- /dev/null +++ b/lib/discovery/gitops/config_fetcher.ex @@ -0,0 +1,209 @@ +defmodule Discovery.GitOps.ConfigFetcher do + @moduledoc """ + Fetches ConfigMap data for CI deploys from different sources: + - git: %{repo, path, rev} + - artifact: %{url} + - reuse_last: true + Returns {:ok, map()} config data suitable for writing as a ConfigMap data section. + Supports env-first layouts: if path points to {env}/{app}, merges {env}/{app}/base/*.yml + and optionally merges {env}/{app}/{app-uid}/configmap.yml when provided via override_path. + """ + + require Logger + + @type config_ref :: %{ + optional(:git) => map(), + optional(:artifact) => map(), + optional(:reuse_last) => boolean() + } + + @spec fetch(String.t(), String.t(), config_ref, String.t()) :: + {:ok, map()} | {:error, String.t()} + def fetch(app_name, _environment, config_ref, working_dir) do + cond do + is_map(config_ref) and Map.has_key?(config_ref, :git) -> + fetch_from_git(config_ref.git, working_dir) + + is_map(config_ref) and Map.has_key?(config_ref, "git") -> + fetch_from_git(config_ref["git"], working_dir) + + is_map(config_ref) and Map.has_key?(config_ref, :artifact) -> + fetch_from_artifact(config_ref.artifact) + + is_map(config_ref) and Map.has_key?(config_ref, "artifact") -> + fetch_from_artifact(config_ref["artifact"]) + + is_map(config_ref) and (config_ref[:reuse_last] || config_ref["reuse_last"]) -> + reuse_last(app_name, working_dir) + + true -> + {:error, "invalid config_ref"} + end + end + + defp fetch_from_git(%{"repo" => repo, "path" => path, "rev" => rev} = m, working_dir) do + fetch_from_git( + %{repo: repo, path: path, rev: rev, override_path: m["override_path"]}, + working_dir + ) + end + + defp fetch_from_git(%{repo: repo, path: path, rev: rev} = m, working_dir) do + tmp_dir = + Path.join([ + working_dir, + ".cfgsrc", + :erlang.unique_integer([:positive]) |> Integer.to_string() + ]) + + File.rm_rf(tmp_dir) + File.mkdir_p!(tmp_dir) + + case System.cmd("git", ["clone", "--no-checkout", repo, tmp_dir], stderr_to_stdout: true) do + {_out, 0} -> + _ = + System.cmd("git", ["-C", tmp_dir, "fetch", "--depth", "1", "origin", rev], + stderr_to_stdout: true + ) + + _ = System.cmd("git", ["-C", tmp_dir, "checkout", "FETCH_HEAD"], stderr_to_stdout: true) + + full = Path.join(tmp_dir, path) + + cond do + File.dir?(full) -> + # Try env-first base merge: {path}/base/*.yml + base_dir = Path.join(full, "base") + + base_maps = + if File.dir?(base_dir) do + base_dir + |> File.ls!() + |> Enum.filter(&String.ends_with?(&1, [".yml", ".yaml"])) + |> Enum.map(&Path.join(base_dir, &1)) + |> Enum.map(&read_yaml_map/1) + |> Enum.filter(&match?({:ok, _}, &1)) + |> Enum.map(fn {:ok, m} -> m end) + else + [] + end + + data = deep_merge_all(base_maps) + + # If there is an override_path (e.g., {env}/{app}/{app-uid}/configmap.yml), merge it on top + data = + case m[:override_path] || m["override_path"] do + nil -> + data + + o when is_binary(o) -> + override_full = Path.join(tmp_dir, o) + + case read_yaml_map(override_full) do + {:ok, om} -> deep_merge(data, om) + _ -> data + end + end + + {:ok, data} + + true -> + # File path; if it is a ConfigMap, extract data; else treat as data map + read_config_yaml(full) + end + + {err, _} -> + Logger.error("config git clone failed: #{err}") + {:error, "git clone failed"} + end + end + + defp fetch_from_artifact(%{"url" => url}), do: fetch_from_artifact(%{url: url}) + + defp fetch_from_artifact(%{url: url}) do + headers = [{"Accept", "application/yaml"}] + + case HTTPoison.get(url, headers, follow_redirect: true) do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + decode_yaml(body) + + {:ok, %HTTPoison.Response{status_code: code, body: body}} -> + {:error, "artifact http #{code}: #{String.slice(to_string(body), 0, 200)}"} + + {:error, reason} -> + {:error, "artifact fetch error: #{inspect(reason)}"} + end + end + + defp reuse_last(app_name, working_dir) do + app_dir = Path.join([working_dir, "apps", app_name]) + + with true <- File.dir?(app_dir), + {:ok, entries} <- File.ls(app_dir), + [latest | _] <- + entries |> Enum.filter(&String.contains?(&1, "#{app_name}-")) |> Enum.sort(:desc), + cfg_path <- Path.join([app_dir, latest, "configmap.yaml"]), + true <- File.exists?(cfg_path), + {:ok, content} <- File.read(cfg_path), + {:ok, map} <- YamlElixir.read_from_string(content, atoms: false) do + data = map["data"] || %{} + {:ok, data} + else + false -> {:error, "no previous deployments for #{app_name}"} + {:error, reason} -> {:error, "read previous config error: #{inspect(reason)}"} + _ -> {:error, "no previous config found"} + end + end + + defp read_yaml_map(path) do + case File.read(path) do + {:ok, content} -> + case YamlElixir.read_from_string(content, atoms: false) do + {:ok, m} when is_map(m) -> + # If full ConfigMap, extract data field + {:ok, + if Map.get(m, "kind") == "ConfigMap" and is_map(m["data"]) do + m["data"] + else + m + end} + + {:error, reason} -> + {:error, reason} + end + + {:error, reason} -> + {:error, reason} + end + end + + defp read_config_yaml(path) do + case File.read(path) do + {:ok, content} -> decode_yaml(content) + {:error, :enoent} -> {:error, "read config error: :enoent (#{path})"} + {:error, reason} -> {:error, "read config error: #{inspect(reason)}"} + end + end + + defp decode_yaml(content) do + case YamlElixir.read_from_string(content, atoms: false) do + {:ok, map} when is_map(map) -> + cond do + Map.get(map, "kind") == "ConfigMap" and is_map(map["data"]) -> {:ok, map["data"]} + true -> {:ok, map} + end + + {:error, reason} -> + {:error, "yaml parse error: #{inspect(reason)}"} + end + end + + defp deep_merge_all([]), do: %{} + defp deep_merge_all([m | rest]), do: Enum.reduce(rest, m, &deep_merge/2) + + defp deep_merge(a, b) when is_map(a) and is_map(b) do + Map.merge(a, b, fn _k, v1, v2 -> deep_merge(v1, v2) end) + end + + defp deep_merge(_a, b), do: b +end diff --git a/lib/discovery/gitops/git_adapter.ex b/lib/discovery/gitops/git_adapter.ex new file mode 100644 index 0000000..d885277 --- /dev/null +++ b/lib/discovery/gitops/git_adapter.ex @@ -0,0 +1,207 @@ +defmodule Discovery.GitOps.GitAdapter do + @moduledoc """ + Git operations for GitOps repository management. + Handles cloning, branching, committing, and pushing to GitOps repos. + """ + + require Logger + + @type git_operation :: :clone | :branch | :commit | :push | :pr + @type git_result :: {:ok, map()} | {:error, String.t()} + + @doc """ + Clones a GitOps repository to a local directory. + """ + @spec clone_repo(String.t(), String.t(), String.t()) :: git_result + def clone_repo(repo_url, local_path, token) do + # Remove existing directory if it exists + File.rm_rf(local_path) + File.mkdir_p!(Path.dirname(local_path)) + + # Use SSH as-is, or HTTPS with token when available + clone_url = build_auth_url(repo_url, token) + + case System.cmd("git", ["clone", clone_url, local_path], stderr_to_stdout: true) do + {output, 0} -> + Logger.info("Successfully cloned repo to #{local_path}") + {:ok, %{path: local_path, output: output}} + + {error_output, _exit_code} -> + Logger.error("Failed to clone repo: #{error_output}") + {:error, "Clone failed: #{error_output}"} + end + end + + @doc """ + Creates a new branch for changes. + """ + @spec create_branch(String.t(), String.t()) :: git_result + def create_branch(repo_path, branch_name) do + with {:ok, _} <- run_git_cmd(repo_path, ["checkout", "-b", branch_name]), + {:ok, _} <- run_git_cmd(repo_path, ["config", "user.email", "discovery@gitops.local"]), + {:ok, _} <- run_git_cmd(repo_path, ["config", "user.name", "Discovery"]) do + {:ok, %{branch: branch_name}} + end + end + + @doc """ + Commits changes with a message. + """ + @spec commit_changes(String.t(), String.t()) :: git_result + def commit_changes(repo_path, message) do + with {:ok, _} <- run_git_cmd(repo_path, ["add", "."]), + {:ok, output} <- run_git_cmd(repo_path, ["commit", "-m", message]) do + {:ok, %{message: message, output: output}} + end + end + + @doc """ + Pushes changes to the remote repository. + """ + @spec push_changes(String.t(), String.t(), String.t()) :: git_result + def push_changes(repo_path, branch, token) do + current_remote = get_remote_url(repo_path) + desired_remote = build_auth_url(current_remote, token) + + # Only rewrite remote if we transitioned to an authenticated HTTPS URL + with {:ok, _} <- maybe_update_remote(repo_path, current_remote, desired_remote), + {:ok, output} <- run_git_cmd(repo_path, ["push", "origin", branch]) do + {:ok, %{branch: branch, output: output}} + end + end + + @doc """ + Creates a pull request (GitHub API). + """ + @spec create_pull_request(String.t(), String.t(), String.t(), String.t(), String.t()) :: + git_result + def create_pull_request(repo_url, token, title, body, branch) do + [owner, repo] = extract_owner_repo(repo_url) + + payload = + %{ + title: title, + body: body, + head: branch, + base: "main" + } + |> Jason.encode!() + + headers = [ + {"Authorization", "token #{token}"}, + {"Accept", "application/vnd.github.v3+json"}, + {"Content-Type", "application/json"} + ] + + url = "https://api.github.com/repos/#{owner}/#{repo}/pulls" + + case HTTPoison.post(url, payload, headers) do + {:ok, %HTTPoison.Response{status_code: 201, body: body}} -> + pr_data = Jason.decode!(body) + {:ok, %{pr_number: pr_data["number"], pr_url: pr_data["html_url"]}} + + {:ok, %HTTPoison.Response{status_code: status, body: body}} -> + Logger.error("Failed to create PR: #{status} - #{body}") + {:error, "PR creation failed: #{body}"} + + {:error, reason} -> + Logger.error("HTTP error creating PR: #{inspect(reason)}") + {:error, "HTTP error: #{inspect(reason)}"} + end + end + + @doc """ + Updates a file in the repository. + """ + @spec update_file(String.t(), String.t(), String.t()) :: :ok | {:error, String.t()} + def update_file(repo_path, file_path, content) do + full_path = Path.join(repo_path, file_path) + + # Ensure directory exists + File.mkdir_p!(Path.dirname(full_path)) + + case File.write(full_path, content) do + :ok -> + Logger.info("Updated file: #{file_path}") + :ok + + {:error, reason} -> + Logger.error("Failed to write file #{file_path}: #{inspect(reason)}") + {:error, "File write failed: #{inspect(reason)}"} + end + end + + @doc """ + Reads a file from the repository. + """ + @spec read_file(String.t(), String.t()) :: {:ok, String.t()} | {:error, String.t()} + def read_file(repo_path, file_path) do + full_path = Path.join(repo_path, file_path) + + case File.read(full_path) do + {:ok, content} -> {:ok, content} + {:error, reason} -> {:error, "File read failed: #{inspect(reason)}"} + end + end + + # Private helper functions + + defp build_auth_url(url, token) do + cond do + is_ssh_url(url) -> + url + + token == nil or token == "" -> + url + + is_https_github_url(url) -> + username = Application.get_env(:discovery, :git_username) || "x-access-token" + String.replace(url, "https://github.com/", "https://#{username}:#{token}@github.com/") + + true -> + url + end + end + + defp is_ssh_url(url) do + String.starts_with?(url, ["git@", "ssh://"]) + end + + defp is_https_github_url(url) do + String.starts_with?(url, "https://github.com/") + end + + defp maybe_update_remote(repo_path, current_remote, desired_remote) do + if current_remote == desired_remote do + {:ok, :unchanged} + else + # Do not override SSH remotes + if is_ssh_url(current_remote) do + {:ok, :ssh_remote_kept} + else + run_git_cmd(repo_path, ["remote", "set-url", "origin", desired_remote]) + end + end + end + + defp get_remote_url(repo_path) do + case System.cmd("git", ["remote", "get-url", "origin"], cd: repo_path) do + {url, 0} -> String.trim(url) + _ -> "" + end + end + + defp extract_owner_repo(repo_url) do + repo_url + |> String.replace("https://github.com/", "") + |> String.replace(".git", "") + |> String.split("/") + end + + def run_git_cmd(repo_path, args) do + case System.cmd("git", args, cd: repo_path, stderr_to_stdout: true) do + {output, 0} -> {:ok, String.trim(output)} + {error_output, _exit_code} -> {:error, String.trim(error_output)} + end + end +end diff --git a/lib/discovery/gitops/gitops_manager.ex b/lib/discovery/gitops/gitops_manager.ex new file mode 100644 index 0000000..f3f2146 --- /dev/null +++ b/lib/discovery/gitops/gitops_manager.ex @@ -0,0 +1,771 @@ +defmodule Discovery.GitOps.GitOpsManager do + @moduledoc """ + Main GitOps manager that orchestrates Git operations, image updates and repository management. + Acts as the primary interface for GitOps operations in Discovery. + """ + + use GenServer + require Logger + + alias Discovery.GitOps.{GitAdapter, RepoLayout, ImageUpdater} + + ## Client functions + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @doc """ + Updates an app's image tag in the GitOps repository. + """ + @spec update_app_image(String.t(), String.t(), String.t()) :: + {:ok, map()} | {:error, String.t()} + def update_app_image(app_name, new_tag, environment \\ "production") do + GenServer.call(__MODULE__, {:update_app_image, app_name, new_tag, environment}, :infinity) + end + + @doc """ + Creates a new app in the GitOps repository. + """ + @spec create_app(String.t(), String.t(), String.t()) :: {:ok, map()} | {:error, String.t()} + def create_app(app_name, image_name, environment \\ "production") do + GenServer.call(__MODULE__, {:create_app, app_name, image_name, environment}, :infinity) + end + + @doc """ + Lists all apps in the GitOps repository. + """ + @spec list_apps() :: {:ok, [String.t()]} | {:error, String.t()} + def list_apps do + GenServer.call(__MODULE__, :list_apps, :infinity) + end + + @doc """ + Gets the current image tag for an app. + """ + @spec get_app_image_tag(String.t(), String.t()) :: {:ok, String.t()} | {:error, String.t()} + def get_app_image_tag(app_name, environment \\ "production") do + GenServer.call(__MODULE__, {:get_app_image_tag, app_name, environment}, :infinity) + end + + @doc """ + Syncs the local minikube/discovery folder to the GitOps repository. + This replaces the S3 upload functionality. + """ + @spec sync_to_gitops(String.t()) :: {:ok, map()} | {:error, String.t()} + def sync_to_gitops(commit_message \\ "Sync Discovery deployment state to GitOps") do + GenServer.call(__MODULE__, {:sync_to_gitops, commit_message}, :infinity) + end + + @doc """ + Syncs a specific app's deployment to the GitOps repository. + """ + @spec sync_app_to_gitops(String.t(), String.t()) :: {:ok, map()} | {:error, String.t()} + def sync_app_to_gitops(app_name, commit_message \\ nil) do + message = commit_message || "Update #{app_name} deployment in GitOps" + GenServer.call(__MODULE__, {:sync_app_to_gitops, app_name, message}, :infinity) + end + + @doc """ + Syncs the entire minikube/discovery folder to the GitOps repository. + This copies all files from minikube/discovery to /tmp/discovery-k8s and pushes to Git. + """ + @spec sync_from_discovery_to_gitops(String.t()) :: {:ok, map()} | {:error, String.t()} + def sync_from_discovery_to_gitops(commit_message \\ "Sync Discovery state to GitOps") do + GenServer.call(__MODULE__, {:sync_from_discovery_to_gitops, commit_message}, :infinity) + end + + @doc """ + Syncs a specific app from minikube/discovery to the GitOps repository. + """ + @spec sync_app_from_discovery_to_gitops(String.t(), String.t()) :: + {:ok, map()} | {:error, String.t()} + def sync_app_from_discovery_to_gitops(app_name, commit_message \\ nil) do + message = commit_message || "Update #{app_name} from Discovery to GitOps" + GenServer.call(__MODULE__, {:sync_app_from_discovery_to_gitops, app_name, message}, :infinity) + end + + @doc """ + Automatically syncs an app after deployment. + This can be called from the deployment workflow to ensure GitOps is always up to date. + """ + @spec auto_sync_after_deployment(String.t()) :: {:ok, map()} | {:error, String.t()} + def auto_sync_after_deployment(app_name) do + commit_message = "Auto-sync: #{app_name} deployment updated" + sync_app_from_discovery_to_gitops(app_name, commit_message) + end + + @doc """ + CI-style deploy: generate uid, write manifests under /tmp working dir, commit and push. + Params: app_name, image, environment, config_ref (git|artifact|reuse_last), idempotency_key(optional) + """ + @spec ci_deploy(String.t(), String.t(), String.t(), map(), String.t() | nil) :: + {:ok, map()} | {:error, String.t()} + def ci_deploy(app_name, image, environment, config_ref, idempotency_key \\ nil) do + GenServer.call( + __MODULE__, + {:ci_deploy, app_name, image, environment, config_ref, idempotency_key}, + :infinity + ) + end + + @doc """ + Returns basic status for a deployment by name (app-uid): presence of manifests. + """ + @spec ci_status(String.t()) :: {:ok, map()} | {:error, String.t()} + def ci_status(deployment_name) do + GenServer.call(__MODULE__, {:ci_status, deployment_name}, :infinity) + end + + ## Server callbacks + + @impl true + def init(opts) do + git_access_token = Application.get_env(:discovery, :git_access_token) + repo_url = Keyword.get(opts, :repo_url, "https://github.com/gamezop/discovery-k8s.git") + token = Keyword.get(opts, :token, git_access_token) + local_path = Keyword.get(opts, :local_path, "/tmp/discovery-k8s") + use_pr = Keyword.get(opts, :use_pr, false) + + # New layout options + # :apps_first | :env_first + write_layout = Keyword.get(opts, :write_layout, :apps_first) + + env_root_map = + Keyword.get(opts, :env_root_map, %{"dev" => "dev", "staging" => "staging", "prod" => "prod"}) + + base_dir_name = Keyword.get(opts, :base_dir_name, "base") + + file_names = + Keyword.get(opts, :file_names, %{ + deployment: "deployment.yml", + configmap: "configmap.yml", + secret: "secret.yml", + service: "service.yml", + ingress: "ingress.yml" + }) + + state = %{ + repo_url: repo_url, + token: token, + local_path: local_path, + use_pr: use_pr, + write_layout: write_layout, + env_root_map: env_root_map, + base_dir_name: base_dir_name, + file_names: file_names + } + + Logger.info("GitOpsManager initialized with repo: #{repo_url} at #{local_path}") + + # Initialize the local directory structure + File.mkdir_p!(local_path) + + {:ok, state} + end + + @impl true + def handle_call({:update_app_image, app_name, new_tag, environment}, _from, state) do + result = do_update_app_image(app_name, new_tag, environment, state) + {:reply, result, state} + end + + @impl true + def handle_call({:create_app, app_name, image_name, environment}, _from, state) do + result = do_create_app(app_name, image_name, environment, state) + {:reply, result, state} + end + + @impl true + def handle_call(:list_apps, _from, state) do + result = do_list_apps(state) + {:reply, result, state} + end + + @impl true + def handle_call({:get_app_image_tag, app_name, environment}, _from, state) do + result = do_get_app_image_tag(app_name, environment, state) + {:reply, result, state} + end + + @impl true + def handle_call({:sync_to_gitops, commit_message}, _from, state) do + result = do_sync_to_gitops(commit_message, state) + {:reply, result, state} + end + + @impl true + def handle_call({:sync_app_to_gitops, app_name, commit_message}, _from, state) do + result = do_sync_app_to_gitops(app_name, commit_message, state) + {:reply, result, state} + end + + @impl true + def handle_call({:sync_from_discovery_to_gitops, commit_message}, _from, state) do + result = do_sync_from_discovery_to_gitops(commit_message, state) + {:reply, result, state} + end + + @impl true + def handle_call({:sync_app_from_discovery_to_gitops, app_name, commit_message}, _from, state) do + result = do_sync_app_from_discovery_to_gitops(app_name, commit_message, state) + {:reply, result, state} + end + + @impl true + def handle_call( + {:ci_deploy, app_name, image, environment, config_ref, idempotency_key}, + _from, + state + ) do + result = do_ci_deploy(app_name, image, environment, config_ref, idempotency_key, state) + {:reply, result, state} + end + + @impl true + def handle_call({:ci_status, deployment_name}, _from, state) do + result = do_ci_status(deployment_name, state) + {:reply, result, state} + end + + # Private implementation functions + + defp do_update_app_image(app_name, new_tag, environment, state) do + with {:ok, _} <- ensure_repo_cloned(state), + {:ok, old_tag} <- + ImageUpdater.get_current_image_tag(state.local_path, app_name, environment), + {:ok, _} <- + ImageUpdater.update_image_tag(state.local_path, app_name, new_tag, environment), + {:ok, commit_result} <- commit_and_push_changes(app_name, old_tag, new_tag, state) do + Logger.info("Successfully updated #{app_name} from #{old_tag} to #{new_tag}") + + {:ok, + %{ + app_name: app_name, + old_tag: old_tag, + new_tag: new_tag, + commit: commit_result + }} + else + {:error, reason} -> + Logger.error("Failed to update app image: #{reason}") + {:error, reason} + end + end + + defp do_create_app(app_name, image_name, environment, state) do + with {:ok, _} <- ensure_repo_cloned(state), + {:ok, _} <- + ImageUpdater.create_deployment_manifest( + state.local_path, + app_name, + image_name, + "latest", + environment + ), + {:ok, commit_result} <- commit_and_push_changes(app_name, nil, "latest", state) do + Logger.info("Successfully created app #{app_name}") + + {:ok, + %{ + app_name: app_name, + image_name: image_name, + commit: commit_result + }} + else + {:error, reason} -> + Logger.error("Failed to create app: #{reason}") + {:error, reason} + end + end + + defp do_list_apps(state) do + with {:ok, _} <- ensure_repo_cloned(state) do + apps = RepoLayout.list_apps(state.local_path) + {:ok, apps} + else + {:error, reason} -> + Logger.error("Failed to list apps: #{reason}") + {:error, reason} + end + end + + defp do_get_app_image_tag(app_name, environment, state) do + with {:ok, _} <- ensure_repo_cloned(state) do + ImageUpdater.get_current_image_tag(state.local_path, app_name, environment) + else + {:error, reason} -> + Logger.error("Failed to get app image tag: #{reason}") + {:error, reason} + end + end + + defp do_sync_to_gitops(commit_message, state) do + with {:ok, _} <- ensure_repo_cloned(state), + {:ok, commit_result} <- commit_and_push_changes("all", nil, nil, state, commit_message) do + Logger.info("Successfully synced entire Discovery state to GitOps") + {:ok, %{message: "Full sync completed", commit: commit_result}} + else + {:error, reason} -> + Logger.error("Failed to sync to GitOps: #{reason}") + {:error, reason} + end + end + + defp do_sync_app_to_gitops(app_name, commit_message, state) do + with {:ok, _} <- ensure_repo_cloned(state), + {:ok, commit_result} <- + commit_and_push_changes(app_name, nil, nil, state, commit_message) do + Logger.info("Successfully synced #{app_name} to GitOps") + {:ok, %{app_name: app_name, commit: commit_result}} + else + {:error, reason} -> + Logger.error("Failed to sync #{app_name} to GitOps: #{reason}") + {:error, reason} + end + end + + defp do_sync_from_discovery_to_gitops(commit_message, state) do + discovery_path = "minikube/discovery" + + with :ok <- ensure_discovery_folder_exists(discovery_path), + :ok <- copy_discovery_to_gitops(discovery_path, state.local_path), + {:ok, _} <- ensure_repo_cloned(state), + {:ok, commit_result} <- commit_and_push_changes("all", nil, nil, state, commit_message) do + Logger.info("Successfully synced entire Discovery state to GitOps") + {:ok, %{message: "Full sync completed", commit: commit_result}} + else + {:error, reason} -> + Logger.error("Failed to sync from Discovery to GitOps: #{reason}") + {:error, reason} + end + end + + defp do_sync_app_from_discovery_to_gitops(app_name, commit_message, state) do + discovery_path = "minikube/discovery" + app_discovery_path = Path.join(discovery_path, app_name) + app_gitops_path = Path.join(state.local_path, app_name) + + with :ok <- ensure_discovery_folder_exists(discovery_path), + :ok <- copy_app_folder(app_discovery_path, app_gitops_path), + {:ok, _} <- ensure_repo_cloned(state), + {:ok, commit_result} <- + commit_and_push_changes(app_name, nil, nil, state, commit_message) do + Logger.info("Successfully synced #{app_name} from Discovery to GitOps") + {:ok, %{app_name: app_name, commit: commit_result}} + else + {:error, reason} -> + Logger.error("Failed to sync #{app_name} from Discovery to GitOps: #{reason}") + {:error, reason} + end + end + + defp do_ci_deploy(app_name, image, environment, config_ref, _idempotency_key, state) do + uid = Discovery.Utils.get_uid() + deployment_name = "#{app_name}-#{uid}" + + app_dir = output_app_dir(state, environment, app_name, deployment_name) + File.mkdir_p!(app_dir) + + # Create app struct for Resource modules + app = %{ + app_name: app_name, + app_image: image, + uid: uid, + # Will be populated from config_ref + config_map: %{}, + app_host: config_ref["app_host"] || "#{app_name}.example.com", + secret_refs: Map.get(config_ref, "secret_refs", []), + app_target_port: 80, + app_container_port: 4000 + } + + with {:ok, _} <- ensure_repo_cloned(state), + {:ok, config_data} <- + Discovery.GitOps.ConfigFetcher.fetch( + app_name, + environment, + config_ref, + state.local_path + ), + app_with_config <- Map.put(app, :config_map, config_data), + :ok <- write_configmap_using_resource(app_dir, app_with_config, state), + :ok <- write_deployment_using_resource(app_dir, app_with_config, state), + :ok <- write_service_using_resource(app_dir, app_with_config, state), + :ok <- upsert_ingress_using_resource(environment, app_name, deployment_name, state), + :ok <- apply_manifests_to_k8s(app_dir, state), + {:ok, commit_result} <- + commit_and_push_changes( + app_name, + nil, + nil, + state, + "feat(ci): deploy #{deployment_name} to #{environment}" + ) do + # Immediately record latest endpoint for clients querying Discovery + write_latest_endpoint_to_metadata_db(app_name, deployment_name, app_with_config.app_host) + + {:ok, + %{ + deployment_name: deployment_name, + app_name: app_name, + environment: environment, + image: image, + git_paths: %{ + deployment: + relative_from_root(state.local_path, Path.join(app_dir, state.file_names.deployment)), + configmap: + relative_from_root(state.local_path, Path.join(app_dir, state.file_names.configmap)), + service: + relative_from_root(state.local_path, Path.join(app_dir, state.file_names.service)) + }, + endpoint: computed_endpoint(app_with_config.app_host, deployment_name), + commit: commit_result + }} + else + {:error, reason} -> {:error, reason} + end + end + + defp computed_endpoint(app_host, deployment_name) do + # Use the actual app_host from CI pipeline instead of hardcoded pattern + "https://#{app_host}/#{deployment_name}" + end + + defp apply_manifests_to_k8s(app_dir, state) do + # Apply manifests directly to K8s for immediate availability + # GitOps will eventually reconcile, but this ensures immediate deployment + manifests = [ + Path.join(app_dir, state.file_names.deployment), + Path.join(app_dir, state.file_names.configmap), + Path.join(app_dir, state.file_names.service) + ] + + # Also apply ingress if it exists + env_root = Map.get(state.env_root_map, "production", "production") + + ingress_path = + Path.join([ + state.local_path, + env_root, + "apps", + Path.basename(app_dir), + state.file_names.ingress + ]) + + manifests = if File.exists?(ingress_path), do: manifests ++ [ingress_path], else: manifests + + results = Enum.map(manifests, &apply_single_manifest/1) + + case Enum.find(results, fn result -> match?({:error, _}, result) end) do + nil -> :ok + {:error, reason} -> {:error, "Failed to apply manifests: #{reason}"} + end + end + + defp apply_single_manifest(manifest_path) do + alias Discovery.Engine.Builder + + with conn when not is_nil(conn) <- Builder.get_conn(), + {:ok, resource_map} <- K8s.Resource.from_file(manifest_path), + operation <- K8s.Client.create(resource_map), + {:ok, _} <- K8s.Client.run(conn, operation) do + Logger.info("Successfully applied manifest: #{manifest_path}") + :ok + else + nil -> + Logger.error("No K8s connection found") + {:error, "No K8s connection"} + + {:error, error} -> + Logger.error("Failed to apply manifest #{manifest_path}: #{inspect(error)}") + {:error, "Failed to apply #{manifest_path}: #{inspect(error)}"} + end + end + + defp write_latest_endpoint_to_metadata_db(app_name, deployment_name, app_host) do + # Mirror structure used by Engine.Builder.update_app_metadata/3 + endpoint = computed_endpoint(app_host, deployment_name) + now = DateTime.utc_now() + + current = + case :ets.lookup(Discovery.Utils.metadata_db(), app_name) do + [{^app_name, m}] -> m + _ -> %{} + end + + updated = + Map.put(current, deployment_name, %{ + "last_updated" => now, + "url" => endpoint + }) + + :ets.insert(Discovery.Utils.metadata_db(), {app_name, updated}) + :ok + end + + defp do_ci_status(deployment_name, state) do + [app_name | _] = String.split(deployment_name, "-") + app_dir = Path.join([state.local_path, "apps", app_name, deployment_name]) + dep = Path.join(app_dir, "deployment.yaml") + cfg = Path.join(app_dir, "configmap.yaml") + + {:ok, + %{ + exists: File.exists?(app_dir), + files: %{ + deployment_yaml: File.exists?(dep), + configmap_yaml: File.exists?(cfg) + }, + paths: %{ + deployment: relative_from_root(state.local_path, dep), + configmap: relative_from_root(state.local_path, cfg) + } + }} + end + + defp write_configmap_using_resource(app_dir, app, state) do + with {:ok, configmap} <- Discovery.Resources.ConfigMap.set_config_map(app), + :ok <- + Discovery.Resources.ConfigMap.write_to_file( + configmap, + Path.join(app_dir, state.file_names.configmap) + ) do + :ok + else + {:error, reason} -> {:error, "ConfigMap creation failed: #{inspect(reason)}"} + end + end + + defp write_deployment_using_resource(app_dir, app, state) do + with {:ok, deployment} <- Discovery.Resources.Deployment.create_deployment(app), + :ok <- + Discovery.Resources.Deployment.write_to_file( + deployment, + Path.join(app_dir, state.file_names.deployment) + ) do + :ok + else + {:error, reason} -> {:error, "Deployment creation failed: #{inspect(reason)}"} + end + end + + defp write_service_using_resource(app_dir, app, state) do + with {:ok, service} <- Discovery.Resources.Service.create_service(app), + :ok <- + Discovery.Resources.Service.write_to_file( + service, + Path.join(app_dir, state.file_names.service) + ) do + :ok + else + {:error, reason} -> {:error, "Service creation failed: #{inspect(reason)}"} + end + end + + defp upsert_ingress_using_resource(environment, app_name, deployment_name, state) do + env_root = Map.get(state.env_root_map, environment, environment) + app_root = Path.join([state.local_path, env_root, app_name]) + ingress_path = Path.join(app_root, state.file_names.ingress) + + # Create a minimal app struct for ingress operations + app = %{ + app_name: app_name, + uid: String.replace(deployment_name, "#{app_name}-", ""), + app_host: "#{app_name}.example.com" + } + + with {:ok, {_ingress_status, ingress}} <- fetch_or_create_ingress(ingress_path, app), + updated_ingress <- add_ingress_path_for_deployment(ingress, deployment_name), + :ok <- Discovery.Resources.Ingress.write_to_file(updated_ingress, ingress_path) do + :ok + else + {:error, reason} -> {:error, "Ingress update failed: #{inspect(reason)}"} + end + end + + defp fetch_or_create_ingress(ingress_path, app) do + case File.exists?(ingress_path) do + true -> + case YamlElixir.read_from_file(ingress_path, atoms: false) do + {:ok, ingress} -> {:ok, {:old_ingress, ingress}} + {:error, _} -> {:error, "Failed to read existing ingress"} + end + + false -> + # Create new ingress using the template + with {:ok, map} <- + YamlElixir.read_from_file("#{:code.priv_dir(:discovery)}/templates/ingress.yml", + atoms: false + ), + map <- put_in(map["metadata"]["name"], app.app_name), + map <- + put_in(map["spec"]["rules"], [ + %{"host" => app.app_host, "http" => %{"paths" => []}} + ]) do + {:ok, {:new_ingress, map}} + else + {:error, _} -> {:error, "Failed to create new ingress"} + end + end + end + + defp add_ingress_path_for_deployment(ingress, deployment_name) do + new_path = %{ + "path" => "/#{deployment_name}(/|$)(.*)", + "pathType" => "Prefix", + "backend" => %{ + "service" => %{ + "name" => deployment_name, + "port" => %{"number" => 80} + } + } + } + + rules = get_in(ingress, ["spec", "rules"]) || [] + rule0 = rules |> List.first() || %{"http" => %{"paths" => []}} + + updated_paths = + (get_in(rule0, ["http", "paths"]) || []) + |> Enum.reject(fn p -> get_in(p, ["backend", "service", "name"]) == deployment_name end) + |> Kernel.++([new_path]) + + updated_rule = put_in(rule0, ["http", "paths"], updated_paths) + + put_in(ingress, ["spec", "rules"], [updated_rule]) + end + + defp output_app_dir(state, environment, app_name, deployment_name) do + case state.write_layout do + :env_first -> + env_root = Map.get(state.env_root_map, environment, environment) + Path.join([state.local_path, env_root, app_name, deployment_name]) + + _ -> + Path.join([state.local_path, "apps", app_name, deployment_name]) + end + end + + defp relative_from_root(root, full) do + case String.replace_prefix(full, root <> "/", "") do + ^full -> full + rel -> rel + end + end + + defp ensure_repo_cloned(state) do + git_dir = Path.join(state.local_path, ".git") + + cond do + File.exists?(git_dir) -> + # Directory is a git repo; try pulling latest. If pull reveals repo is broken, reclone. + case GitAdapter.run_git_cmd(state.local_path, ["pull", "origin", "main"]) do + {:ok, _} -> + {:ok, :already_cloned} + + {:error, reason} -> + Logger.warning("Failed to pull latest changes: #{reason}") + + if String.contains?(String.downcase(reason), "not a git repository") do + Logger.warning("Local path is not a valid git repo; recloning...") + File.rm_rf(state.local_path) + GitAdapter.clone_repo(state.repo_url, state.local_path, state.token) + else + {:ok, :already_cloned} + end + end + + File.exists?(state.local_path) -> + # Path exists but is not a git repo; clean and clone afresh. + Logger.warning("Local path exists without .git; recloning repo at #{state.local_path}") + File.rm_rf(state.local_path) + GitAdapter.clone_repo(state.repo_url, state.local_path, state.token) + + true -> + # Path doesn't exist; fresh clone + GitAdapter.clone_repo(state.repo_url, state.local_path, state.token) + end + end + + defp ensure_discovery_folder_exists(discovery_path) do + if File.exists?(discovery_path) do + :ok + else + Logger.warning("Discovery folder #{discovery_path} does not exist") + {:error, "Discovery folder not found: #{discovery_path}"} + end + end + + defp copy_discovery_to_gitops(discovery_path, gitops_path) do + # Remove existing gitops folder and recreate + File.rm_rf(gitops_path) + File.mkdir_p!(gitops_path) + + # Copy all files and folders from discovery to gitops + copy_directory_contents(discovery_path, gitops_path) + :ok + end + + defp copy_app_folder(app_discovery_path, app_gitops_path) do + # Remove existing app folder in gitops and recreate + File.rm_rf(app_gitops_path) + File.mkdir_p!(app_gitops_path) + + # Copy app folder contents + copy_directory_contents(app_discovery_path, app_gitops_path) + :ok + end + + defp copy_directory_contents(source, destination) do + case File.ls(source) do + {:ok, items} -> + Enum.each(items, fn item -> + source_path = Path.join(source, item) + dest_path = Path.join(destination, item) + + if File.dir?(source_path) do + # Copy directory recursively + File.mkdir_p!(dest_path) + copy_directory_contents(source_path, dest_path) + else + # Copy file + File.cp!(source_path, dest_path) + end + end) + + {:error, reason} -> + Logger.error("Failed to list directory #{source}: #{reason}") + {:error, "Failed to list directory: #{reason}"} + end + end + + defp commit_and_push_changes(app_name, old_tag, new_tag, state, custom_message \\ nil) do + message = + custom_message || RepoLayout.get_commit_message(app_name, old_tag || "none", new_tag) + + if state.use_pr do + # Create PR workflow + branch_name = "update-#{app_name}-#{new_tag}-#{:rand.uniform(10000)}" + + with {:ok, _} <- GitAdapter.create_branch(state.local_path, branch_name), + {:ok, _} <- GitAdapter.commit_changes(state.local_path, message), + {:ok, _} <- GitAdapter.push_changes(state.local_path, branch_name, state.token), + {:ok, pr_result} <- + GitAdapter.create_pull_request( + state.repo_url, + state.token, + RepoLayout.get_pr_title(app_name, new_tag), + RepoLayout.get_pr_body(app_name, old_tag || "none", new_tag), + branch_name + ) do + {:ok, %{type: :pr, pr: pr_result}} + end + else + # Direct push to main + with {:ok, _} <- GitAdapter.commit_changes(state.local_path, message), + {:ok, _} <- GitAdapter.push_changes(state.local_path, "main", state.token) do + {:ok, %{type: :commit, message: message}} + end + end + end +end diff --git a/lib/discovery/gitops/image_updater.ex b/lib/discovery/gitops/image_updater.ex new file mode 100644 index 0000000..42fc73e --- /dev/null +++ b/lib/discovery/gitops/image_updater.ex @@ -0,0 +1,231 @@ +defmodule Discovery.GitOps.ImageUpdater do + @moduledoc """ + Updates Docker image tags in Kubernetes deployment manifests. + Handles YAML parsing, image tag replacement, and validation. + """ + + require Logger + alias Discovery.GitOps.RepoLayout + alias Discovery.Utils + + @type update_result :: {:ok, map()} | {:error, String.t()} + + @doc """ + Updates the image tag in a deployment manifest. + """ + @spec update_image_tag(String.t(), String.t(), String.t(), String.t()) :: update_result + def update_image_tag(repo_path, app_name, new_tag, environment \\ "production") do + config = RepoLayout.get_app_config(app_name, environment) + + with {:ok, content} <- read_deployment_file(repo_path, config.deployment_file), + {:ok, updated_content} <- replace_image_tag(content, new_tag), + :ok <- validate_yaml(updated_content), + :ok <- write_deployment_file(repo_path, config.deployment_file, updated_content) do + Logger.info("Successfully updated #{app_name} image to #{new_tag}") + {:ok, %{app_name: app_name, new_tag: new_tag, file: config.deployment_file}} + end + end + + @doc """ + Gets the current image tag from a deployment manifest. + """ + @spec get_current_image_tag(String.t(), String.t(), String.t()) :: + {:ok, String.t()} | {:error, String.t()} + def get_current_image_tag(repo_path, app_name, environment \\ "production") do + config = RepoLayout.get_app_config(app_name, environment) + + with {:ok, content} <- read_deployment_file(repo_path, config.deployment_file), + {:ok, parsed} <- parse_yaml(content), + {:ok, tag} <- extract_image_tag(parsed) do + {:ok, tag} + end + end + + @doc """ + Creates a new deployment manifest for an app. + """ + @spec create_deployment_manifest(String.t(), String.t(), String.t(), String.t(), String.t()) :: + update_result + def create_deployment_manifest( + repo_path, + app_name, + image_name, + tag, + environment \\ "production" + ) do + config = RepoLayout.get_app_config(app_name, environment) + namespace = RepoLayout.get_namespace(app_name, environment) + + manifest = generate_deployment_manifest(app_name, image_name, tag, namespace) + + with :ok <- validate_yaml(manifest), + :ok <- write_deployment_file(repo_path, config.deployment_file, manifest) do + Logger.info("Successfully created deployment manifest for #{app_name}") + {:ok, %{app_name: app_name, file: config.deployment_file}} + end + end + + # Private helper functions + + defp read_deployment_file(repo_path, file_path) do + full_path = Path.join(repo_path, file_path) + + case File.read(full_path) do + {:ok, content} -> {:ok, content} + {:error, :enoent} -> {:error, "Deployment file not found: #{file_path}"} + {:error, reason} -> {:error, "Failed to read deployment file: #{inspect(reason)}"} + end + end + + defp write_deployment_file(repo_path, file_path, content) do + full_path = Path.join(repo_path, file_path) + + # Ensure directory exists + File.mkdir_p!(Path.dirname(full_path)) + + case File.write(full_path, content) do + :ok -> :ok + {:error, reason} -> {:error, "Failed to write deployment file: #{inspect(reason)}"} + end + end + + defp replace_image_tag(content, new_tag) do + with {:ok, parsed} <- parse_yaml(content), + {:ok, updated} <- update_image_in_parsed(parsed, new_tag), + {:ok, updated_content} <- encode_yaml(updated) do + {:ok, updated_content} + end + end + + defp update_image_in_parsed(parsed, new_tag) do + case get_in(parsed, ["spec", "template", "spec", "containers"]) do + nil -> + {:error, "No containers found in deployment"} + + containers when is_list(containers) -> + updated_containers = + Enum.map(containers, fn container -> + case container["image"] do + nil -> + container + + current_image -> + # Extract image name without tag + image_name = String.split(current_image, ":") |> List.first() + Map.put(container, "image", "#{image_name}:#{new_tag}") + end + end) + + updated_parsed = + put_in(parsed, ["spec", "template", "spec", "containers"], updated_containers) + + {:ok, updated_parsed} + + _ -> + {:error, "Invalid containers format"} + end + end + + defp extract_image_tag(parsed) do + case get_in(parsed, ["spec", "template", "spec", "containers"]) do + [container | _] -> + case container["image"] do + nil -> + {:error, "No image found in container"} + + image -> + case String.split(image, ":") do + [_image_name, tag] -> {:ok, tag} + _ -> {:error, "No tag found in image: #{image}"} + end + end + + _ -> + {:error, "No containers found in deployment"} + end + end + + defp generate_deployment_manifest(app_name, image_name, tag, namespace) do + """ + apiVersion: apps/v1 + kind: Deployment + metadata: + name: #{app_name} + namespace: #{namespace} + labels: + app: #{app_name} + spec: + replicas: 3 + selector: + matchLabels: + app: #{app_name} + template: + metadata: + labels: + app: #{app_name} + spec: + containers: + - name: #{app_name} + image: #{image_name}:#{tag} + ports: + - containerPort: 3000 + env: + - name: NODE_ENV + value: "production" + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: /health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 5 + """ + end + + defp parse_yaml(content) do + case YamlElixir.read_from_string(content, atoms: false) do + {:ok, parsed} -> {:ok, parsed} + {:error, reason} -> {:error, "YAML parse error: #{inspect(reason)}"} + end + end + + defp encode_yaml(parsed) do + # Use the existing Utils.to_yml function + temp_file = "/tmp/temp_manifest_#{:rand.uniform(10000)}.yml" + + case Utils.to_yml(parsed, temp_file) do + :ok -> + case File.read(temp_file) do + {:ok, content} -> + File.rm(temp_file) + {:ok, content} + + {:error, reason} -> + File.rm(temp_file) + {:error, "Failed to read temp file: #{inspect(reason)}"} + end + + {:error, reason} -> + {:error, "YAML encode error: #{inspect(reason)}"} + end + end + + defp validate_yaml(content) do + case parse_yaml(content) do + {:ok, _} -> :ok + {:error, reason} -> {:error, "Invalid YAML: #{inspect(reason)}"} + end + end +end diff --git a/lib/discovery/gitops/repo_layout.ex b/lib/discovery/gitops/repo_layout.ex new file mode 100644 index 0000000..cedb15e --- /dev/null +++ b/lib/discovery/gitops/repo_layout.ex @@ -0,0 +1,161 @@ +defmodule Discovery.GitOps.RepoLayout do + @moduledoc """ + Manages GitOps repository layout and file path conventions. + Handles app-specific paths, environment overlays, and file naming. + """ + + @type app_config :: %{ + app_name: String.t(), + environment: String.t(), + base_path: String.t(), + deployment_file: String.t(), + service_file: String.t(), + ingress_file: String.t(), + configmap_file: String.t() + } + + @doc """ + Gets the repository layout configuration for an app and environment. + """ + @spec get_app_config(String.t(), String.t()) :: app_config + def get_app_config(app_name, environment \\ "production") do + base_path = "apps/#{app_name}" + + %{ + app_name: app_name, + environment: environment, + base_path: base_path, + deployment_file: "#{base_path}/deployment.yaml", + service_file: "#{base_path}/service.yaml", + ingress_file: "#{base_path}/ingress.yaml", + configmap_file: "#{base_path}/configmap.yaml" + } + end + + @doc """ + Gets the namespace for an app and environment. + """ + @spec get_namespace(String.t(), String.t()) :: String.t() + def get_namespace(_app_name, _environment \\ "production") do + # Keeping namespace discovery for all deployments done + # via discovery + "dicovery" + # case environment do + # "production" -> app_name + # env -> "#{app_name}-#{env}" + # end + end + + @doc """ + Gets the image name pattern for an app. + """ + @spec get_image_pattern(String.t()) :: String.t() + def get_image_pattern(app_name) do + # Default pattern - can be configured per app + "ghcr.io/your-org/#{app_name}" + end + + @doc """ + Gets the commit message for an image update. + """ + @spec get_commit_message(String.t(), String.t(), String.t()) :: String.t() + def get_commit_message(app_name, old_tag, new_tag) do + "feat: update #{app_name} image from #{old_tag} to #{new_tag}" + end + + @doc """ + Gets the PR title for an image update. + """ + @spec get_pr_title(String.t(), String.t()) :: String.t() + def get_pr_title(app_name, new_tag) do + "Update #{app_name} to #{new_tag}" + end + + @doc """ + Gets the PR body for an image update. + """ + @spec get_pr_body(String.t(), String.t(), String.t()) :: String.t() + def get_pr_body(app_name, old_tag, new_tag) do + """ + ## Image Update + + **App:** #{app_name} + **Previous:** #{old_tag} + **New:** #{new_tag} + + This PR updates the Docker image for #{app_name} to version #{new_tag}. + + ### Changes + - Updated deployment manifest with new image tag + - No other configuration changes + + ### Testing + - [ ] Image builds successfully + - [ ] Deployment is healthy + - [ ] No breaking changes detected + """ + end + + @doc """ + Validates if a file path is within the allowed app structure. + """ + @spec validate_file_path(String.t(), String.t()) :: :ok | {:error, String.t()} + def validate_file_path(app_name, file_path) do + expected_base = "apps/#{app_name}/" + + case String.starts_with?(file_path, expected_base) do + true -> :ok + false -> {:error, "File path #{file_path} is not within app directory #{expected_base}"} + end + end + + @doc """ + Gets all file paths for an app. + """ + @spec get_app_files(String.t()) :: [String.t()] + def get_app_files(app_name) do + config = get_app_config(app_name) + + [ + config.deployment_file, + config.service_file, + config.ingress_file, + config.configmap_file + ] + end + + @doc """ + Gets the directory path for an app. + """ + @spec get_app_directory(String.t()) :: String.t() + def get_app_directory(app_name) do + "apps/#{app_name}" + end + + @doc """ + Checks if an app directory exists in the repository. + """ + @spec app_exists?(String.t(), String.t()) :: boolean() + def app_exists?(repo_path, app_name) do + app_dir = Path.join(repo_path, get_app_directory(app_name)) + File.exists?(app_dir) and File.dir?(app_dir) + end + + @doc """ + Lists all apps in the repository. + """ + @spec list_apps(String.t()) :: [String.t()] + def list_apps(repo_path) do + apps_dir = Path.join(repo_path, "apps") + + case File.exists?(apps_dir) do + true -> + apps_dir + |> File.ls!() + |> Enum.filter(&File.dir?(Path.join(apps_dir, &1))) + + false -> + [] + end + end +end diff --git a/lib/discovery/resources/deployment.ex b/lib/discovery/resources/deployment.ex index 68247e9..22d1de8 100644 --- a/lib/discovery/resources/deployment.ex +++ b/lib/discovery/resources/deployment.ex @@ -7,58 +7,82 @@ defmodule Discovery.Resources.Deployment do import Discovery.K8Config + @template_path "#{:code.priv_dir(:discovery)}/templates/deploy.yml" + @spec create_deployment(DeployUtils.app()) :: {:error, any()} | {:ok, map()} def create_deployment(app) do - with {:ok, map} <- - "#{:code.priv_dir(:discovery)}/templates/deploy.yml" - |> YamlElixir.read_from_file(atoms: false), - map <- put_in(map["apiVersion"], api_version(:deployment)), - map <- put_in(map, ["metadata", "name"], "#{app.app_name}-#{app.uid}"), - map <- put_in(map, ["metadata", "annotations", "app_id"], "#{app.app_name}"), - map <- add_service_account(map), - map <- - put_in(map, ["spec", "selector", "matchLabels", "app"], "#{app.app_name}-#{app.uid}"), - map <- - put_in(map, ["spec", "template", "spec", "imagePullSecrets"], [ - %{"name" => Application.get_env(:discovery, :image_pull_secrets)} - ]), - map <- - put_in( - map, - ["spec", "template", "metadata", "labels", "app"], - "#{app.app_name}-#{app.uid}" - ) do - deployment_container = map["spec"]["template"]["spec"]["containers"] |> hd - - deployment_container = - put_in(deployment_container["envFrom"], [ - %{ - "configMapRef" => %{"name" => "#{app.app_name}-#{app.uid}"} - } - ]) - - deployment_container = put_in(deployment_container["image"], "#{app.app_image}") - deployment_container = put_in(deployment_container["name"], "#{app.app_name}") - - deployment_container = - put_in(deployment_container["ports"], [ - %{ - "containerPort" => app.app_container_port, - "name" => "#{app.app_name}-port", - "protocol" => "TCP" - } - ]) - - deployment_container = put_in(deployment_container["resources"], resources()) - - map = put_in(map["spec"]["template"]["spec"]["containers"], [deployment_container]) - - {:ok, map} + with {:ok, map} <- read_deployment_template(), + {:ok, updated_map} <- update_deployment_map(map, app), + {:ok, updated_map} <- update_container(updated_map, app), + {:ok, final_map} <- update_labels(updated_map, app) do + {:ok, final_map} else {:error, _} -> {:error, "error in creating deployment config"} end end + defp read_deployment_template do + "#{@template_path}" + |> YamlElixir.read_from_file(atoms: false) + end + + defp update_deployment_map(map, app) do + map = + map + |> put_in(["apiVersion"], api_version(:deployment)) + |> put_in(["metadata", "name"], "#{app.app_name}-#{app.uid}") + |> put_in(["metadata", "annotations", "app_id"], "#{app.app_name}") + |> add_service_account() + |> put_in(["spec", "selector", "matchLabels", "app"], "#{app.app_name}-#{app.uid}") + |> put_in(["spec", "template", "spec", "nodeSelector"], %{ + "kubernetes.io/arch" => Application.get_env(:discovery, :kubernetes_arch) + }) + |> put_in(["spec", "template", "spec", "imagePullSecrets"], [ + %{"name" => Application.get_env(:discovery, :image_pull_secrets)} + ]) + + {:ok, map} + end + + defp update_container(map, app) do + [deployment_container | _] = get_in(map, ["spec", "template", "spec", "containers"]) + + # Build envFrom with one ConfigMapRef and 0..N SecretRefs + env_from = + [%{"configMapRef" => %{"name" => "#{app.app_name}-#{app.uid}"}}] ++ + Enum.map(Map.get(app, :secret_refs, []), fn secret_name -> + %{"secretRef" => %{"name" => secret_name}} + end) + + deployment_container = + deployment_container + |> put_in(["envFrom"], env_from) + |> put_in(["image"], "#{app.app_image}") + |> put_in(["name"], "#{app.app_name}") + |> put_in(["ports"], [ + %{ + "containerPort" => app.app_container_port, + "name" => "#{app.app_name}-port", + "protocol" => "TCP" + } + ]) + |> put_in(["resources"], resources()) + + map = + map + |> put_in(["spec", "template", "spec", "containers"], [deployment_container]) + + {:ok, map} + end + + defp update_labels(map, app) do + map = + map + |> put_in(["spec", "template", "metadata", "labels", "app"], "#{app.app_name}-#{app.uid}") + + {:ok, map} + end + @spec write_to_file(map, String.t()) :: :ok def write_to_file(map, location) do Utils.to_yml(map, location) diff --git a/lib/discovery/resources/ingress.ex b/lib/discovery/resources/ingress.ex index e689601..3ffac25 100644 --- a/lib/discovery/resources/ingress.ex +++ b/lib/discovery/resources/ingress.ex @@ -23,9 +23,15 @@ defmodule Discovery.Resources.Ingress do def add_ingress_path(current_ingress_map, app) do new_path = %{ "path" => "/#{app.uid}(/|$)(.*)", + # "pathType" => "ImplementationSpecific", + "pathType" => "Prefix", "backend" => %{ - "serviceName" => "#{app.app_name}-#{app.uid}", - "servicePort" => 80 + "service" => %{ + "name" => "#{app.app_name}-#{app.uid}", + "port" => %{ + "number" => 80 + } + } } } @@ -101,7 +107,7 @@ defmodule Discovery.Resources.Ingress do |> get_in(["http", "paths"]) |> Enum.map(fn path_details -> path_details - |> get_in(["backend", "serviceName"]) + |> get_in(["backend", "service", "name"]) end) _ -> diff --git a/lib/discovery/s3_uploader.ex b/lib/discovery/s3_uploader.ex new file mode 100644 index 0000000..1c696c0 --- /dev/null +++ b/lib/discovery/s3_uploader.ex @@ -0,0 +1,83 @@ +defmodule Discovery.S3Uploader do + @moduledoc """ + Module for S3 related functions + """ + + require Logger + + @doc """ + Uploads file to S3 bucket + + Expects: + - local path of the file + - bucket name + - filename_file_path: path to be saved in bucket + """ + @spec upload_file(String.t(), String.t(), String.t()) :: + :ok | {:error, String.t()} + def upload_file(path, bucket, upload_file_path) do + with {:ok, file} <- File.read(path), + operation <- + ExAws.S3.put_object(bucket, upload_file_path, file), + {:ok, _resp} <- ExAws.request(operation) do + :ok + else + {:error, reason} -> + Logger.error("error in uploading to S3 #{inspect(reason)}") + {:error, "error in uploading to S3 #{inspect(reason)}"} + end + end + + def download_contents(bucket) do + with operation <- ExAws.S3.list_objects(bucket), + {:ok, %{body: %{contents: contents}}} <- ExAws.request(operation) do + contents + |> Enum.each(fn content -> + data = download(bucket, content.key) + + Path.dirname(content.key) + |> File.mkdir_p() + + {:ok, file} = File.open(content.key, [:write, :utf8, :binary]) + IO.write(file, data) + end) + else + {:error, reason} -> + Logger.error("error in downloading from S3 #{inspect(reason)}") + {:error, "error in downloading from S3 #{inspect(reason)}"} + end + end + + def delete_content(bucket, object) do + with operation <- ExAws.S3.list_objects(bucket, prefix: object), + {:ok, %{body: %{contents: contents}}} <- ExAws.request(operation), + objects_list <- contents |> Enum.reduce([], fn x, acc -> [x.key | acc] end), + operation <- ExAws.S3.delete_multiple_objects(bucket, objects_list), + {:ok, resp} <- ExAws.request(operation) do + resp + end + end + + @doc """ + Tags the object for bucket level customisations + """ + @spec tag_object(String.t(), String.t(), map()) :: :ok | {:error, String.t()} + def tag_object(bucket, upload_file_path, tags) do + with operation <- + ExAws.S3.put_object_tagging(bucket, upload_file_path, tags), + {:ok, _resp} <- ExAws.request(operation) do + :ok + else + {:error, reason} -> + Logger.error("error in object tagging in #{bucket} #{inspect(reason)}") + {:error, "error in object tagging in #{bucket} #{inspect(reason)}"} + end + end + + def download(bucket, file_path) do + ExAws.S3.download_file(bucket, file_path, :memory) + |> ExAws.stream!() + |> Enum.map(& &1) + |> List.first() + end +end diff --git a/lib/discovery/utils.ex b/lib/discovery/utils.ex index 4a43beb..acbbde8 100644 --- a/lib/discovery/utils.ex +++ b/lib/discovery/utils.ex @@ -40,12 +40,19 @@ defmodule Discovery.Utils do IO.puts(IO.ANSI.format([:red_background, :black, inspect(term)])) end - # @spec to_yml(any) :: String.t() + @doc """ + On every write to the config file, we upload the file to S3 + """ @spec to_yml(map, String.t()) :: :ok def to_yml(map, location) do yml = Yamlix.dump(map, false) {:ok, io} = File.open(location, [:write, :utf8]) IO.write(io, yml) + + bucket = Application.get_env(:discovery, :discovery_bucket) + Discovery.S3Uploader.upload_file(location, bucket, location) + + :ok end end diff --git a/lib/discovery_web/controllers/ci_controller.ex b/lib/discovery_web/controllers/ci_controller.ex new file mode 100644 index 0000000..17cb593 --- /dev/null +++ b/lib/discovery_web/controllers/ci_controller.ex @@ -0,0 +1,45 @@ +defmodule DiscoveryWeb.CiController do + use DiscoveryWeb, :controller + alias Discovery.GitOps.GitOpsManager + + @deploy_params %{ + app_name: [type: :string, required: true], + image: [type: :string, required: true], + environment: [type: :string, required: true], + config_ref: [type: :map, required: true], + idempotency_key: [type: :string, required: false] + } + + @spec deploy(Plug.Conn.t(), map()) :: Plug.Conn.t() + def deploy(conn, params) do + with {:ok, params} <- Tarams.cast(params, @deploy_params), + {:ok, result} <- + GitOpsManager.ci_deploy( + params.app_name, + params.image, + params.environment, + params.config_ref, + params[:idempotency_key] + ) do + json(conn, %{success: true, data: result}) + else + {:error, reason} -> + conn |> put_status(400) |> json(%{success: false, error: reason}) + end + end + + @status_params %{ + deployment_name: [type: :string, required: true] + } + + @spec status(Plug.Conn.t(), map()) :: Plug.Conn.t() + def status(conn, params) do + with {:ok, params} <- Tarams.cast(params, @status_params), + {:ok, result} <- GitOpsManager.ci_status(params.deployment_name) do + json(conn, %{success: true, data: result}) + else + {:error, reason} -> + conn |> put_status(400) |> json(%{success: false, error: reason}) + end + end +end diff --git a/lib/discovery_web/controllers/gitops_controller.ex b/lib/discovery_web/controllers/gitops_controller.ex new file mode 100644 index 0000000..93961c3 --- /dev/null +++ b/lib/discovery_web/controllers/gitops_controller.ex @@ -0,0 +1,228 @@ +defmodule DiscoveryWeb.GitOpsController do + use DiscoveryWeb, :controller + alias Discovery.GitOps.GitOpsManager + + @doc """ + Updates an app's image tag in the GitOps repository. + """ + @spec update_image(Plug.Conn.t(), map()) :: Plug.Conn.t() + def update_image(conn, params) do + update_params = %{ + app_name: [type: :string, required: true], + new_tag: [type: :string, required: true], + environment: [type: :string, default: "production"] + } + + with {:ok, params} <- Tarams.cast(params, update_params), + {:ok, result} <- + GitOpsManager.update_app_image(params.app_name, params.new_tag, params.environment) do + json(conn, %{ + success: true, + message: "Successfully updated #{params.app_name} to #{params.new_tag}", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to update image: #{reason}" + }) + end + end + + @doc """ + Creates a new app in the GitOps repository. + """ + @spec create_app(Plug.Conn.t(), map()) :: Plug.Conn.t() + def create_app(conn, params) do + create_params = %{ + app_name: [type: :string, required: true], + image_name: [type: :string, required: true], + environment: [type: :string, default: "production"] + } + + with {:ok, params} <- Tarams.cast(params, create_params), + {:ok, result} <- + GitOpsManager.create_app(params.app_name, params.image_name, params.environment) do + json(conn, %{ + success: true, + message: "Successfully created app #{params.app_name}", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to create app: #{reason}" + }) + end + end + + @doc """ + Lists all apps in the GitOps repository. + """ + @spec list_apps(Plug.Conn.t(), map()) :: Plug.Conn.t() + def list_apps(conn, _params) do + case GitOpsManager.list_apps() do + {:ok, apps} -> + json(conn, %{ + success: true, + data: %{apps: apps} + }) + + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to list apps: #{reason}" + }) + end + end + + @doc """ + Gets the current image tag for an app. + """ + @spec get_image_tag(Plug.Conn.t(), map()) :: Plug.Conn.t() + def get_image_tag(conn, params) do + tag_params = %{ + app_name: [type: :string, required: true], + environment: [type: :string, default: "production"] + } + + with {:ok, params} <- Tarams.cast(params, tag_params), + {:ok, tag} <- GitOpsManager.get_app_image_tag(params.app_name, params.environment) do + json(conn, %{ + success: true, + data: %{ + app_name: params.app_name, + environment: params.environment, + current_tag: tag + } + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to get image tag: #{reason}" + }) + end + end + + @doc """ + Syncs the entire Discovery state to the GitOps repository. + """ + @spec sync_to_gitops(Plug.Conn.t(), map()) :: Plug.Conn.t() + def sync_to_gitops(conn, params) do + sync_params = %{ + commit_message: [type: :string, default: "Sync Discovery deployment state to GitOps"] + } + + with {:ok, params} <- Tarams.cast(params, sync_params), + {:ok, result} <- GitOpsManager.sync_to_gitops(params.commit_message) do + json(conn, %{ + success: true, + message: "Successfully synced to GitOps", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to sync to GitOps: #{reason}" + }) + end + end + + @doc """ + Syncs a specific app's deployment to the GitOps repository. + """ + @spec sync_app_to_gitops(Plug.Conn.t(), map()) :: Plug.Conn.t() + def sync_app_to_gitops(conn, params) do + sync_params = %{ + app_name: [type: :string, required: true], + commit_message: [type: :string, default: nil] + } + + with {:ok, params} <- Tarams.cast(params, sync_params), + {:ok, result} <- GitOpsManager.sync_app_to_gitops(params.app_name, params.commit_message) do + json(conn, %{ + success: true, + message: "Successfully synced #{params.app_name} to GitOps", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to sync app to GitOps: #{reason}" + }) + end + end + + @doc """ + Syncs the entire minikube/discovery folder to the GitOps repository. + """ + @spec sync_from_discovery_to_gitops(Plug.Conn.t(), map()) :: Plug.Conn.t() + def sync_from_discovery_to_gitops(conn, params) do + sync_params = %{ + commit_message: [type: :string, default: "Sync Discovery state to GitOps"] + } + + with {:ok, params} <- Tarams.cast(params, sync_params), + {:ok, result} <- GitOpsManager.sync_from_discovery_to_gitops(params.commit_message) do + json(conn, %{ + success: true, + message: "Successfully synced Discovery state to GitOps", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to sync from Discovery to GitOps: #{reason}" + }) + end + end + + @doc """ + Syncs a specific app from minikube/discovery to the GitOps repository. + """ + @spec sync_app_from_discovery_to_gitops(Plug.Conn.t(), map()) :: Plug.Conn.t() + def sync_app_from_discovery_to_gitops(conn, params) do + sync_params = %{ + app_name: [type: :string, required: true], + commit_message: [type: :string, default: nil] + } + + with {:ok, params} <- Tarams.cast(params, sync_params), + {:ok, result} <- + GitOpsManager.sync_app_from_discovery_to_gitops(params.app_name, params.commit_message) do + json(conn, %{ + success: true, + message: "Successfully synced #{params.app_name} from Discovery to GitOps", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to sync app from Discovery to GitOps: #{reason}" + }) + end + end +end diff --git a/lib/discovery_web/router.ex b/lib/discovery_web/router.ex index 8701501..a44fc6e 100644 --- a/lib/discovery_web/router.ex +++ b/lib/discovery_web/router.ex @@ -35,6 +35,22 @@ defmodule DiscoveryWeb.Router do post "/deploy-build", BaseController, :deploy_build delete "/delete-app", BaseController, :delete_app delete "/delete-deployment", BaseController, :delete_deployment + + # GitOps endpoints + post "/gitops/update-image", GitOpsController, :update_image + post "/gitops/create-app", GitOpsController, :create_app + get "/gitops/apps", GitOpsController, :list_apps + get "/gitops/image-tag", GitOpsController, :get_image_tag + post "/gitops/sync", GitOpsController, :sync_to_gitops + post "/gitops/sync-app", GitOpsController, :sync_app_to_gitops + post "/gitops/sync-from-discovery", GitOpsController, :sync_from_discovery_to_gitops + post "/gitops/sync-app-from-discovery", GitOpsController, :sync_app_from_discovery_to_gitops + + # CI endpoints + scope "/ci" do + post "/deploy", CiController, :deploy + get "/status", CiController, :status + end end # Enables LiveDashboard only for development diff --git a/makefile b/makefile index 6627f83..d6f1957 100644 --- a/makefile +++ b/makefile @@ -1,4 +1,4 @@ -VERSION_DEV = 0.2.1 +VERSION_DEV = 0.2.5 VERSION_PROD = 0.1.0 commit: diff --git a/mix.exs b/mix.exs index 16f8541..4a2c8c5 100644 --- a/mix.exs +++ b/mix.exs @@ -1,12 +1,10 @@ defmodule Discovery.MixProject do use Mix.Project - @version "0.2.1" - def project do [ app: :discovery, - version: @version, + version: "0.3.0", elixir: "~> 1.12", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), @@ -56,6 +54,10 @@ defmodule Discovery.MixProject do {:quantum, "~> 3.0"}, {:yaml_elixir, "~> 2.9.0"}, {:tarams, "~> 1.6.1"}, + {:ex_aws, "~> 2.1", override: true}, + {:ex_aws_s3, "~> 2.0"}, + {:sweet_xml, "~> 0.6"}, + {:httpoison, "~> 1.8"}, {:yamlix, git: "https://github.com/ghostdsb/yamlix.git", branch: "master"} # {:yamlix, path: "../ext-modules/yamlix"} ] diff --git a/mix.lock b/mix.lock index 363f5a5..6834112 100644 --- a/mix.lock +++ b/mix.lock @@ -10,6 +10,8 @@ "crontab": {:hex, :crontab, "1.1.11", "4028ced51b813a5061f85b689d4391ef0c27550c8ab09aaf139e4295c3d93ea4", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "ecb045f9ac14a3e2990e54368f70cdb6e2f2abafc5bc329d6c31f0c74b653787"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "earmark_parser": {:hex, :earmark_parser, "1.4.25", "2024618731c55ebfcc5439d756852ec4e85978a39d0d58593763924d9a15916f", [:mix], [], "hexpm", "56749c5e1c59447f7b7a23ddb235e4b3defe276afc220a6227237f3efe83f51e"}, + "ex_aws": {:hex, :ex_aws, "2.4.1", "d1dc8965d1dc1c939dd4570e37f9f1d21e047e4ecd4f9373dc89cd4e45dce5ef", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "803387db51b4e91be4bf0110ba999003ec6103de7028b808ee9b01f28dbb9eee"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.4.0", "ce8decb6b523381812798396bc0e3aaa62282e1b40520125d1f4eff4abdff0f4", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "85dda6e27754d94582869d39cba3241d9ea60b6aa4167f9c88e309dc687e56bb"}, "ex_doc": {:hex, :ex_doc, "0.28.4", "001a0ea6beac2f810f1abc3dbf4b123e9593eaa5f00dd13ded024eae7c523298", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bf85d003dd34911d89c8ddb8bda1a958af3471a274a4c2150a9c01c78ac3f8ed"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "floki": {:hex, :floki, "0.32.1", "dfe3b8db3b793939c264e6f785bca01753d17318d144bd44b407fb3493acaa87", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "d4b91c713e4a784a3f7b1e3cc016eefc619f6b1c3898464222867cafd3c681a3"}, @@ -41,7 +43,8 @@ "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, "quantum": {:hex, :quantum, "3.5.0", "8d2c5ba68c55991e8975aca368e3ab844ba01f4b87c4185a7403280e2c99cf34", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.14 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_registry, "~> 0.2", [hex: :telemetry_registry, repo: "hexpm", optional: false]}], "hexpm", "cab737d1d9779f43cb1d701f46dd05ea58146fd96238d91c9e0da662c1982bb6"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, + "sweet_xml": {:hex, :sweet_xml, "0.7.3", "debb256781c75ff6a8c5cbf7981146312b66f044a2898f453709a53e5031b45b", [:mix], [], "hexpm", "e110c867a1b3fe74bfc7dd9893aa851f0eed5518d0d7cad76d7baafd30e4f5ba"}, "tailwind": {:hex, :tailwind, "0.1.5", "5561bed6c114434415077972f6d291e7d43b258ef0ee756bda1ead7293811f61", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "3be21a0ddec7fc29b323ee72bed7516078a2787f7b142e455698a2209296e2a5"}, "tarams": {:hex, :tarams, "1.6.1", "55549de2464b8d0548697fc9ca4df1d7bc5d6db19ec2b8467ac59ba9d7d27ed8", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:valdi, "~> 0.3", [hex: :valdi, repo: "hexpm", optional: false]}], "hexpm", "6d703e79e059be2f2163a5ded35120afca983fa3012412045c1e24c59dbd36a5"}, "telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"}, diff --git a/priv/templates/deploy.yml b/priv/templates/deploy.yml index 7ef3fe4..b2502a1 100644 --- a/priv/templates/deploy.yml +++ b/priv/templates/deploy.yml @@ -21,6 +21,8 @@ spec: labels: app: APP_NAME-UID spec: + nodeSelector: + kubernetes.io/arch: amd64 serviceAccountName: discovery-sa containers: - name: APP_NAME diff --git a/priv/templates/service.yml b/priv/templates/service.yml index 3b30a01..779009d 100644 --- a/priv/templates/service.yml +++ b/priv/templates/service.yml @@ -5,6 +5,7 @@ metadata: name: APP_NAME-UID namespace: discovery spec: + type: NodePort ports: - port: 80 targetPort: 3245