Feat/git adapter

.aws/buckets.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -x
+# awslocal s3 mb s3://test-static.quizzop.com 
+aws --endpoint-url=http://127.0.0.1:4566 s3api create-bucket --bucket dev-discovery --create-bucket-configuration LocationConstraint=ap-south-1
+set +x

.gitignore

@@ -37,4 +37,9 @@ npm-debug.log
 
 /minikube/
 
-/.vscode/
+/.vscode
+
+/.env
+
+example_workflow.sh
+test_gitops.exs

.tool-versions

@@ -1,2 +1,3 @@
-elixir 1.12.2-otp-24
-erlang 24.0
+elixir 1.18.2-otp-27
+erlang 27.2
+nodejs 19.8.1

assets/.tool-versions

@@ -0,0 +1 @@
+nodejs 19.8.1

config/config.exs

@@ -20,6 +20,16 @@ config :logger, :console,
   format: "$time $metadata[$level] $message\n",
   metadata: [:request_id]
 
+# Aws s3 config
+config :ex_aws,
+  json_codec: Jason
+
+config :ex_aws, :s3,
+  scheme: "https://",
+  region: "ap-south-1",
+  host: "s3-ap-south-1.amazonaws.com",
+  port: 443
+
 # Use Jason for JSON parsing in Phoenix
 config :phoenix, :json_library, Jason
 
@@ -44,15 +54,19 @@ config :discovery,
   service_account: "discovery-sa",
   use_external_ingress_class: true,
   ingress_class: "nginx-external",
-  image_pull_secrets: "dockerhub-auth-discovery"
+  image_pull_secrets: "dockerhub-auth-discovery",
+  kubernetes_arch: "amd64"
 
 config :discovery, :api_version,
   config_map: "v1",
   deployment: "apps/v1",
-  ingress: "networking.k8s.io/v1beta1",
+  ingress: "networking.k8s.io/v1",
   namespace: "v1",
   service: "v1"
 
+config :discovery,
+  git_username: "ghostdsb"
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/dev.exs

@@ -12,13 +12,13 @@ config :discovery, DiscoveryWeb.Endpoint,
   code_reloader: true,
   check_origin: false,
   watchers: [
-    node: [
-      "node_modules/webpack/bin/webpack.js",
-      "--mode",
-      "development",
-      "--watch-stdin",
-      cd: Path.expand("../assets", __DIR__)
-    ]
+    # node: [
+    #   "node_modules/webpack/bin/webpack.js",
+    #   "--mode",
+    #   "development",
+    #   "--watch-stdin",
+    #   cd: Path.expand("../assets", __DIR__)
+    # ]
   ]
 
 # ## SSL Support
@@ -56,6 +56,17 @@ config :discovery, DiscoveryWeb.Endpoint,
     ]
   ]
 
+# uncomment for testing on localhost for quizzop question upload module
+config :ex_aws, :s3,
+  scheme: "http://",
+  region: "ap-south-1",
+  host: "localhost",
+  port: 4566
+
+config :discovery,
+  discovery_bucket: "dev-discovery",
+  discovery_bucket_url: "https://dev-discovery.gamezop.com"
+
 # Do not include metadata nor timestamps in development logs
 config :logger, :console, format: "[$level] $message\n"
 
@@ -71,11 +82,19 @@ config :discovery, :base_url, "http://localhost:4000"
 config :discovery,
   connection_method: :kube_config,
   namespace: "discovery",
-  service_account: "discovery-service-account",
+  ## "discovery-service-account",
+  service_account: "default",
   resources: %{
     limits: %{cpu: "500m", memory: "500Mi"},
     requests: %{cpu: "100m", memory: "300Mi"}
   },
   image_pull_secrets: "ghostdsb-auth-discovery",
   use_external_ingress_class: false,
   use_service_account: true
+
+# config :discovery, :api_version,
+#   config_map: "v1",
+#   deployment: "apps/v1",
+#   ingress: "networking.k8s.io/v1beta1",
+#   namespace: "v1",
+#   service: "v1"

config/runtime.exs

@@ -28,4 +28,18 @@ if config_env() == :prod or config_env() == :develop do
   #
   # Then you can assemble a release by calling `mix release`.
   # See `mix help release` for more information.
+  aws_access_id = System.fetch_env!("AWS_ACCESS_KEY_ID")
+  aws_access_key = System.fetch_env!("AWS_SECRET_ACCESS_KEY")
+  discovery_bucket = System.fetch_env!("DISCOVERY_BUCKET")
+  discovery_bucket_url = System.fetch_env!("DISCOVERY_BUCKET_URL")
+  git_access_token = System.fetch_env!("GITHUB_REPO_TOKEN")
+
+  config :ex_aws,
+    access_key_id: aws_access_id,
+    secret_access_key: aws_access_key
+
+  config :discovery,
+    discovery_bucket: discovery_bucket,
+    discovery_bucket_url: discovery_bucket_url,
+    git_access_token: git_access_token
 end

config/test.exs

@@ -8,3 +8,17 @@ config :discovery, DiscoveryWeb.Endpoint,
 
 # Print only warnings and errors during test
 config :logger, level: :warn
+
+config :ex_aws, :s3,
+  scheme: "http://",
+  region: "ap-south-1",
+  host: "localhost",
+  port: 4566
+
+config :ex_aws,
+  access_key_id: "secret",
+  secret_access_key: "secret"
+
+config :discovery,
+  discovery_bucket: "discovery_bucket",
+  discovery_bucket_url: "discovery_bucket_url"

dev.Dockerfile

@@ -20,8 +20,8 @@ ENV MIX_ENV="${MIX_ENV}"
 
 # install mix dependencies
 COPY mix.exs mix.lock ./
-RUN mix deps.get --only $MIX_ENV
-RUN mkdir config
+COPY config config
+RUN mix do deps.get, deps.compile
 
 # copy compile-time config files before we compile dependencies
 # to ensure any relevant config change will trigger the dependencies

dev.env

@@ -1,2 +1,7 @@
-KUBERNETES_SERVICE_HOST="192.168.49.2"
-KUBERNETES_SERVICE_PORT=8443
+export KUBERNETES_SERVICE_HOST="192.168.49.2"
+export KUBERNETES_SERVICE_PORT=8443
+export AWS_ACCESS_KEY_ID=key
+export AWS_SECRET_ACCESS_KEY=secret
+export DISCOVERY_BUCKET=dev-discovery
+export DISCOVERY_BUCKET_URL=https://dev-discovery.gamezop.com
+export GITHUB_REPO_TOKEN=secret

docker-compose.yaml

@@ -0,0 +1,20 @@
+version: "3.9"
+
+services:
+  localstack:
+    image: localstack/localstack
+    ports:
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+      - "127.0.0.1:4510-4559:4510-4559"  # external services port range
+      - "127.0.0.1:53:53"                # DNS config (only required for Pro)
+      - "127.0.0.1:53:53/udp"            # DNS config (only required for Pro)
+      - "127.0.0.1:443:443"              # LocalStack HTTPS Gateway (only required for Pro)
+    environment:
+      - SERVICES=s3
+      - DEBUG=${DEBUG-}
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - DEFAULT_REGION=ap-south-1
+      - AWS_ACCESS_KEY_ID=key
+      - AWS_SECRET_ACCESS_KEY=secret
+    volumes:
+      - ./.aws:/docker-entrypoint-initaws.d

dockerfiles/Dockerfile

@@ -0,0 +1,91 @@
+FROM hexpm/elixir:1.14.0-erlang-25.1-alpine-3.15.6 as build
+
+# install build dependencies
+RUN apk add --no-cache build-base git npm python3 curl
+
+
+# prepare build dir
+WORKDIR /app
+
+# install hex + rebar
+RUN mix local.hex --force && \
+    mix local.rebar --force
+
+# set build ENV
+ARG MIX_ENV
+ENV MIX_ENV="${MIX_ENV}"
+
+# install mix dependencies
+COPY mix.exs mix.lock ./
+RUN --mount=type=secret,id=github_token \
+    GITHUB_TOKEN=$(cat /run/secrets/github_token) && \
+    echo "machine github.com login $GITHUB_TOKEN" > ~/.netrc && \
+    mix deps.get --only ${MIX_ENV}
+RUN mkdir config
+
+# copy compile-time config files before we compile dependencies
+# to ensure any relevant config change will trigger the dependencies
+# to be re-compiled.
+COPY config/config.exs config/$MIX_ENV.exs config/
+RUN mix deps.compile
+
+COPY priv priv
+# COPY assets assets
+
+# compile and build release
+COPY lib lib
+
+RUN mix compile
+# changes to config/runtime.exs don't require recompiling the code
+COPY config/runtime.exs config/
+
+# uncomment COPY if rel/ exists
+# COPY rel rel
+RUN MIX_ENV=${MIX_ENV} mix release
+
+# prepare release image
+FROM alpine:3.15.6 AS app
+
+RUN apk add --no-cache openssl ncurses-libs libgcc libstdc++ git npm python3 curl
+
+ARG MIX_ENV
+ENV USER="elixir"
+
+WORKDIR "/home/${USER}/app"
+
+# Creates an unprivileged user to be used exclusively to run the Phoenix app
+# RUN \
+  # addgroup \
+  #  -g 1000 \
+  #  -S "${USER}" \
+  # && adduser \
+  #  -s /bin/sh \
+  #  -u 1000 \
+  #  -G "${USER}" \
+  #  -h "/home/${USER}" \
+  #  -D "${USER}" \
+  # && su "${USER}"
+
+# Creates super privileged user
+
+RUN chown root:root /home/elixir/app
+
+# Everything from this line onwards will run in the context of the unprivileged user.
+# USER "${USER}"
+
+# Everything from this line onwards will run in the context of the super privileged user.
+
+USER root:root
+
+COPY --from=build --chown=root:root /app/_build/"${MIX_ENV}"/rel/discovery ./
+
+COPY ./dockerfiles/entrypoint.sh .
+
+ARG MIX_ENV
+ENV MIX_ENV="${MIX_ENV}"
+
+# ARG POD_A_RECORD
+# ENV POD_A_RECORD="${echo ${POD_IP} | sed 's/\\./-/g'}"
+
+# Run the Phoenix app
+CMD ["sh", "./entrypoint.sh"]

dockerfiles/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+bin="bin/discovery"
+
+# start the elixir application
+exec "$bin" "start"

lib/discovery/application.ex

@@ -8,12 +8,15 @@ defmodule Discovery.Application do
   alias Discovery.Controller.DeploymentController
   alias Discovery.Deploy.DeployManager
   alias Discovery.Engine.Builder
+  alias Discovery.GitOps.GitOpsManager
   alias Discovery.Scheduler
   alias Discovery.Utils
 
   require Logger
 
   def start(_type, _args) do
+    git_access_token = Application.get_env(:discovery, :git_access_token)
+
     children = [
       # Start the Telemetry supervisor
       DiscoveryWeb.Telemetry,
@@ -24,6 +27,23 @@ defmodule Discovery.Application do
       {Builder, []},
       {DeploymentController, []},
       {DeployManager, []},
+      {GitOpsManager,
+       [
+         repo_url: "git@github.com:gamezop/discovery-k8s.git",
+         token: git_access_token,
+         local_path: "/tmp/discovery-k8s",
+         use_pr: false,
+         write_layout: :env_first,
+         env_root_map: %{"dev" => "dev", "staging" => "staging", "prod" => "prod"},
+         base_dir_name: "base",
+         file_names: %{
+           deployment: "deploy.yml",
+           configmap: "configmap.yml",
+           secret: "secret.yml",
+           service: "service.yml",
+           ingress: "ingress.yml"
+         }
+       ]},
       Scheduler
       # Start a worker by calling: Discovery.Worker.start_link(arg)
       # {Discovery.Worker, arg}