From 3dac01603688e1813ba8cf392adc661e25e3f8aa Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 01:51:19 +0000 Subject: [PATCH 01/20] Add custom providers documentation for the plugin system Document how to use custom providers via GitHub repos, NPM packages, or local paths. Covers the ProviderInterface, supported source formats, caching behavior, and a full example implementation. Also updates the API reference to mention custom providers under providerStrategy. Co-Authored-By: Claude Opus 4.6 --- .../04-api-reference.mdx | 5 +- .../07-custom-providers.mdx | 182 ++++++++++++++++++ 2 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 docs/03-github-orchestrator/06-advanced-topics/07-custom-providers.mdx diff --git a/docs/03-github-orchestrator/04-api-reference.mdx b/docs/03-github-orchestrator/04-api-reference.mdx index 6a46b5c3..6b1b85d1 100644 --- a/docs/03-github-orchestrator/04-api-reference.mdx +++ b/docs/03-github-orchestrator/04-api-reference.mdx @@ -90,8 +90,9 @@ attempt to read them from current directory's git repo (e.g branch, commit SHA, providerStrategy ``` -Specifies the Cloud Provider to use for Orchestrator jobs. Accepted values: `aws`, `k8s`, -`local-docker`, `local`. +Specifies the Cloud Provider to use for Orchestrator jobs. Built-in values: `aws`, `k8s`, +`local-docker`, `local`. You can also specify a custom provider via GitHub URL, NPM package, or +local path (see [Custom Providers](advanced-topics/custom-providers)). ```bash - containerCpu diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-custom-providers.mdx b/docs/03-github-orchestrator/06-advanced-topics/07-custom-providers.mdx new file mode 100644 index 00000000..b855a223 --- /dev/null +++ b/docs/03-github-orchestrator/06-advanced-topics/07-custom-providers.mdx @@ -0,0 +1,182 @@ +# Custom Providers + +Orchestrator uses a plugin system that lets you extend it with custom providers. A **provider** is a +pluggable backend that controls where and how your builds run. Built-in providers include `aws`, +`k8s`, `local-docker`, and `local`. + +With custom providers, you can point `providerStrategy` at a GitHub repository, NPM package, or +local path and Orchestrator will dynamically load it at runtime. + +## Using a Custom Provider + +Set `providerStrategy` to a provider source instead of a built-in name: + +```yaml +# GitHub repository +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: 'https://github.com/your-org/my-provider' + targetPlatform: StandaloneLinux64 + +# GitHub shorthand +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: 'your-org/my-provider' + targetPlatform: StandaloneLinux64 + +# Specific branch +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: 'your-org/my-provider@develop' + targetPlatform: StandaloneLinux64 +``` + +### Supported Source Formats + +| Format | Example | +| ------------------------------------- | -------------------------------------------------------- | +| GitHub HTTPS URL | `https://github.com/user/repo` | +| GitHub URL with branch | `https://github.com/user/repo/tree/main` | +| GitHub URL with branch and path | `https://github.com/user/repo/tree/main/src/my-provider` | +| GitHub shorthand | `user/repo` | +| GitHub shorthand with branch | `user/repo@develop` | +| GitHub shorthand with branch and path | `user/repo@develop/src/my-provider` | +| GitHub SSH | `git@github.com:user/repo.git` | +| NPM package | `my-provider-package` | +| Scoped NPM package | `@scope/my-provider` | +| Local relative path | `./my-local-provider` | +| Local absolute path | `/path/to/provider` | + +## Creating a Custom Provider + +A provider is a module that exports a class implementing the `ProviderInterface`. The module must +have an entry point at one of: `index.js`, `index.ts`, `src/index.js`, `src/index.ts`, +`lib/index.js`, `lib/index.ts`, or `dist/index.js`. + +### Required Methods + +Every provider must implement these 7 methods: + +```typescript +interface ProviderInterface { + setupWorkflow( + buildGuid: string, + buildParameters: BuildParameters, + branchName: string, + defaultSecretsArray: { + ParameterKey: string; + EnvironmentVariable: string; + ParameterValue: string; + }[], + ): Promise; + + runTaskInWorkflow( + buildGuid: string, + image: string, + commands: string, + mountdir: string, + workingdir: string, + environment: OrchestratorEnvironmentVariable[], + secrets: OrchestratorSecret[], + ): Promise; + + cleanupWorkflow( + buildParameters: BuildParameters, + branchName: string, + defaultSecretsArray: { + ParameterKey: string; + EnvironmentVariable: string; + ParameterValue: string; + }[], + ): Promise; + + garbageCollect( + filter: string, + previewOnly: boolean, + olderThan: Number, + fullCache: boolean, + baseDependencies: boolean, + ): Promise; + + listResources(): Promise; + listWorkflow(): Promise; + watchWorkflow(): Promise; +} +``` + +### Example Implementation + +```typescript +// index.ts +export default class MyProvider { + constructor(private buildParameters: any) {} + + async setupWorkflow(buildGuid, buildParameters, branchName, defaultSecretsArray) { + // Initialize your build environment + } + + async runTaskInWorkflow(buildGuid, image, commands, mountdir, workingdir, environment, secrets) { + // Execute the build task in your environment + return 'Build output'; + } + + async cleanupWorkflow(buildParameters, branchName, defaultSecretsArray) { + // Tear down resources after the build + } + + async garbageCollect(filter, previewOnly, olderThan, fullCache, baseDependencies) { + // Clean up old resources + return 'Garbage collection complete'; + } + + async listResources() { + // Return active resources + return []; + } + + async listWorkflow() { + // Return running workflows + return []; + } + + async watchWorkflow() { + // Stream logs from a running workflow + return ''; + } +} +``` + +## How It Works + +When `providerStrategy` is set to a value that doesn't match a built-in provider name, Orchestrator +will: + +1. **Detect the source type** — GitHub URL, NPM package, or local path. +2. **Fetch the provider** — For GitHub repos, the repository is cloned (shallow, depth 1) into a + `.provider-cache/` directory. Cached repos are automatically updated on subsequent runs. +3. **Load the module** — The entry point is imported and the default export is used. +4. **Validate the interface** — All 7 required methods are checked. If any are missing, loading + fails. +5. **Fallback** — If loading fails for any reason, Orchestrator logs the error and falls back to the + local provider so your pipeline doesn't break. + +## Caching + +GitHub repositories are cached in the `.provider-cache/` directory, keyed by owner, repo, and +branch. On subsequent runs the loader checks for updates and pulls them automatically. + +### Environment Variables + +| Variable | Default | Description | +| -------------------- | ----------------- | --------------------------------------- | +| `PROVIDER_CACHE_DIR` | `.provider-cache` | Custom cache directory for cloned repos | +| `GIT_TIMEOUT` | `30000` | Git operation timeout in milliseconds | + +## Best Practices + +- **Pin a branch or tag** — Use `user/repo@v1.0` or a specific branch to avoid unexpected changes. +- **Test locally first** — Use a local path during development before publishing. +- **Handle errors gracefully** — Your provider methods should throw clear errors so Orchestrator can + log them and fall back if needed. +- **Keep it lightweight** — The provider module is loaded at runtime. Minimize dependencies to keep + startup fast. From d706850fdffc9874086e26aa938163391240c290 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 05:20:57 +0000 Subject: [PATCH 02/20] Comprehensive orchestrator documentation overhaul - Restructure providers into dedicated section with overview, custom providers, community providers (with GitHub edit link), and GitHub/GitLab integration pages - Rewrite API reference with proper tables, all missing parameters (orchestratorRepoName, githubOwner, allowDirtyBuild, postBuildSteps, preBuildSteps, customHookFiles, customCommandHooks, useCleanupCron), and environment variables (AWS_FORCE_PROVIDER, PURGE_REMOTE_BUILDER_CACHE, ORCHESTRATOR_AWS_STACK_WAIT_TIME, GIT_PRIVATE_TOKEN) - Document premade rclone hooks and Steam deployment hooks - Add S3/rclone workspace locking documentation - Tighten language across all pages for clarity - Add ASCII diagrams to introduction, caching, logging, and config override - Add tasteful emoji to section headers - Rename "Game-CI vs Orchestrator" to "Standard Game-CI vs Orchestrator Mode" - Remove outdated Deno section from command line docs - Improve examples with proper tables, workflow snippets, and cross-links Co-Authored-By: Claude Opus 4.6 --- .../01-introduction.mdx | 142 +++---- .../02-game-ci-vs-orchestrator.mdx | 61 ++- .../03-examples/01-command-line.mdx | 66 ++-- .../03-examples/02-github-examples/02-aws.mdx | 73 ++-- .../02-github-examples/03-kubernetes.mdx | 68 ++-- .../04-api-reference.mdx | 364 +++++++----------- .../06-advanced-topics/01-caching.mdx | 66 ++-- .../02-retained-workspace.mdx | 34 +- .../03-garbage-collection.mdx | 33 +- .../04-configuration-override.mdx | 45 ++- .../04-custom-hooks/01-custom-job.mdx | 19 +- .../04-custom-hooks/03-command-hooks.mdx | 27 +- .../04-custom-hooks/04-container-hooks.mdx | 25 +- .../05-premade-container-jobs.mdx | 60 +-- .../06-advanced-topics/05-logging.mdx | 35 +- .../06-advanced-topics/06-secrets.mdx | 15 +- .../07-providers/01-overview.mdx | 57 +++ .../02-custom-providers.mdx} | 0 .../07-providers/03-community-providers.mdx | 47 +++ .../07-providers/04-github-integration.mdx | 41 ++ .../07-providers/05-gitlab-integration.mdx | 30 ++ .../07-providers/_category_.yaml | 5 + .../09-gitlab/01-introduction-gitlab.mdx | 3 - .../09-gitlab/_category_.yaml | 5 - .../10-github/01-github-checks.mdx | 7 - .../10-github/_category_.yaml | 5 - 26 files changed, 776 insertions(+), 557 deletions(-) create mode 100644 docs/03-github-orchestrator/06-advanced-topics/07-providers/01-overview.mdx rename docs/03-github-orchestrator/06-advanced-topics/{07-custom-providers.mdx => 07-providers/02-custom-providers.mdx} (100%) create mode 100644 docs/03-github-orchestrator/06-advanced-topics/07-providers/03-community-providers.mdx create mode 100644 docs/03-github-orchestrator/06-advanced-topics/07-providers/04-github-integration.mdx create mode 100644 docs/03-github-orchestrator/06-advanced-topics/07-providers/05-gitlab-integration.mdx create mode 100644 docs/03-github-orchestrator/06-advanced-topics/07-providers/_category_.yaml delete mode 100644 docs/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx delete mode 100644 docs/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml delete mode 100644 docs/03-github-orchestrator/06-advanced-topics/10-github/01-github-checks.mdx delete mode 100644 docs/03-github-orchestrator/06-advanced-topics/10-github/_category_.yaml diff --git a/docs/03-github-orchestrator/01-introduction.mdx b/docs/03-github-orchestrator/01-introduction.mdx index 3ae13efb..3e2af6d9 100644 --- a/docs/03-github-orchestrator/01-introduction.mdx +++ b/docs/03-github-orchestrator/01-introduction.mdx @@ -1,104 +1,78 @@ # Introduction -## Concept - What Does Orchestrator Do +## What Does Orchestrator Do? -**Orchestrator enables you to run, build and test (Unity) projects in the cloud. You can start jobs -from the command line, the "Workbench" GUI in the Unity Editor or from GitHub Actions.** +**Orchestrator runs Unity builds on cloud infrastructure.** Start jobs from GitHub Actions, the +command line, or any CI system. Orchestrator provisions a cloud environment, sends your project to +be built, and streams results back. -**Orchestrator will automatically provision an environment at a Cloud Provider such as GCP and AWS. -It will then send the project to be built and/or tested depending on your workflow configuration.** - -**Orchestrator is especially useful for game development because it supports large projects. -Orchestrator provides first class support for the Unity game engine.** - -Orchestrator uses git to track and syncronize your projects and uses native cloud services such as -AWS Fargate and Kubernetes to run your jobs. Other version control systems are not actively -supported. - -## Why Orchestrator? - -1. **Orchestrator is flexible and elastic** - 1. _You can balance your use of high-performance and cost saving modes._ Configurable cost/speed - effeciency - 2. _Works great for projects of almost any size, from AAA projects and assets to micro projects_ - 3. _Extended configuration options._ More control over disk size, memory and CPU - 4. _Easily scale all job resources as your project grows_ e.g storage, CPU and memory -2. **Scale fully on demand from zero (not in use) to many concurrent jobs.** Benefits from - "pay-for-what-you-use" cloud billing models. We have made an effort to make sure that it costs - you nothing (aside from artifact and cache storage) while there are no builds running (no - guarantees) -3. **Easy setup and configuration** -4. **Run custom jobs** and extend the system for any workload - -## Why not orchestrator? - -1. Your project is small in size. Below 5GB Orchestrator should not be needed. -2. You already have dedicated servers running you can use. - -Although the speed of a CI pipelines is an important metric to consider, there are real challenges -for game development pipelines. - -This solution prefers convenience, ease of use, scalability, throughput and flexibility. - -Faster solutions exist, but would all involve self-hosted hardware with an immediate local cache of -the large project files and working directory and a dedicated server. - -# Orchestrator Release Status - -Orchestrator is in "active development" ⚠️🔨 +``` + Your Machine / CI Cloud Provider + ┌──────────────┐ git push ┌──────────────────┐ + │ │ ──────────────────► │ ☁️ AWS Fargate │ + │ GitHub │ │ ☁️ Kubernetes │ + │ Actions │ ◄──────────────── │ 🐳 Local Docker │ + │ │ build artifacts │ │ + └──────────────┘ └──────────────────┘ + │ │ + │ Orchestrator handles: │ + │ • Provisioning │ + │ • Git sync + LFS │ + │ • Caching │ + │ • Cleanup │ + └─────────────────────────────────────┘ +``` -Orchestrator overall release status: `preview` This means some APIs may change, features are still -being added but the minimum feature set works and is stable. +Orchestrator supports large projects with first-class Unity support and native cloud services like +AWS Fargate and Kubernetes. -Release Stages: `experimental` ➡️ `preview` ➡️ `full release` +## ✅ Why Orchestrator? -You must use a provider with Orchestrator, each provider's release status is described below. This -indicates the stability and support for orchestrator features and workflows. +1. **Flexible and elastic** — balance speed and cost, configure CPU, memory, and disk per build +2. **Scale from zero** — no idle servers, pay only while builds run +3. **Easy setup** — minimal configuration to get started +4. **Extensible** — run custom jobs, add hooks, or bring your own provider via the plugin system -### Supported Orchestrator Platforms +## ❌ When You Don't Need It -```md -| Cloud Provider Platform | Release Status | -| ----------------------- | ------------------ | -| Kubernetes | ✔️ preview release | -| AWS | ✔️ full release | -| GCP | ⚠ Considered | -| Azure | ⚠ Considered | -``` +- Your project is under 5 GB — standard GitHub runners should work fine +- You have dedicated build servers already running -_Note for Kubernetes support:_ _Usually the cluster needs to be up and running at all times, as -starting up a cluster is slow._ _Use Google Cloud's Kubernetes Autopilot you can scale down to the -free tier automatically while not in use._ _Kubernetes support currently requires cloud storage, -only S3 support is built-in._ - -```md -| Git Platform | Release Status | -| --------------------- | ------------------ | -| GitHub | ✔️ full release | -| GitLab | ✔️ preview release | -| Command Line | ✔️ preview release | -| Any Git repository | ✔️ preview release | -| Any Git automation/Ci | ✔️ preview release | -``` +## 📦 Release Status -## External Links +Orchestrator is in **active development** ⚠️ -### Orchestrator Releases +Overall status: `preview` — core features are stable, some APIs may change. -[Game CI Releases - GitHub](https://github.com/game-ci/unity-builder/releases) _Packaged and -released with game-ci's unity-builder module._ +Release stages: `experimental` → `preview` → `full release` -### Open Incoming Pull Requests +### Provider Support -[Orchestrator PRs - GitHub](https://github.com/game-ci/unity-builder/pulls?q=is%3Apr+orchestrator) +| Cloud Provider | Release Status | +| -------------- | ------------------ | +| AWS Fargate | ✔️ Full release | +| Kubernetes | ✔️ Preview release | +| GCP | ⚠️ Considered | +| Azure | ⚠️ Considered | -### 💬Suggestions and 🐛Bugs (GitHub Issues): +See [Providers](advanced-topics/providers/overview) for details on each provider. -[Game CI Issues - GitHub](https://github.com/game-ci/unity-builder/labels/orchestrator) +### Platform Support -### Community +| Platform | Release Status | +| ------------------ | ------------------ | +| GitHub Actions | ✔️ Full release | +| GitLab CI | ✔️ Preview release | +| Command Line | ✔️ Preview release | +| Any Git repository | ✔️ Preview release | +| Any CI system | ✔️ Preview release | -**Share your feedback with us!** +## 🔗 External Links -- [**Discord Channel**](https://discord.com/channels/710946343828455455/789631903157583923) -- [**Feedback Form**](https://forms.gle/3Wg1gGf9FnZ72RiJ9) +- [Releases](https://github.com/game-ci/unity-builder/releases) — packaged with + game-ci/unity-builder +- [Pull Requests](https://github.com/game-ci/unity-builder/pulls?q=is%3Apr+orchestrator) — open + orchestrator PRs +- [Issues](https://github.com/game-ci/unity-builder/labels/orchestrator) — bugs and feature requests +- [Discord](https://discord.com/channels/710946343828455455/789631903157583923) — community chat +- [Feedback Form](https://forms.gle/3Wg1gGf9FnZ72RiJ9) — share your experience diff --git a/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx b/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx index 369bc10e..61e46eb5 100644 --- a/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx +++ b/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx @@ -1,42 +1,41 @@ -# Game-CI vs Orchestrator +# Standard Game-CI vs Orchestrator Mode -# Standard Game-CI (Use Cases) +## 🎮 Standard Game-CI -The Game CI core is a maintained set of docker images that can be used to run workloads in many -scenarios. +Game CI provides Docker images and GitHub Actions for running Unity workflows on the build server +resources provided by your CI platform (GitHub, GitLab, Circle CI). -Game CI also provides specific GitHub actions for running workflows on GitHub. And a similar -workflow for running Game CI on GitLab and Circle CI. _All of these options use the build server -resources provided by those systems, this can be a constraint or very convenient depending on the -size of your project and the workloads you need to run._ +**Best for:** Small to medium projects that fit within GitHub's resource limits. -# Orchestrator (Use Cases) +## ☁️ Orchestrator Mode -## Sending Builds to the cloud +Orchestrator sends builds to cloud infrastructure (AWS Fargate, Kubernetes) instead of running on +the CI runner itself. This is useful when: -You may want to take advantage of cloud resources for lots of reasons (scale, speed, cost, -flexibility) or may want to start remote builds from the command line without slowing down your -development machine. Orchestrator can help you do this. +- Your project **exceeds disk space limits** on GitHub-hosted runners +- You need **more CPU or memory** than the CI platform provides +- You want to **scale to many concurrent builds** without managing servers -This may be a preference, more effecient or you may want to use systems that struggle to handle -large game development projects (GitHub being a good example). +``` + Standard Game-CI Orchestrator Mode -### Large GitHub Projects + ┌──────────────┐ ┌──────────────┐ ┌────────────┐ + │ GitHub │ │ GitHub │ ───► │ Cloud │ + │ Runner │ │ Runner │ │ Container │ + │ (builds │ │ (dispatches │ ◄─── │ (builds │ + │ locally) │ │ only) │ │ remotely) │ + └──────────────┘ └──────────────┘ └────────────┘ + ~14 GB disk Configurable CPU, memory, disk + Fixed resources Scales to zero when idle +``` -GitHub Actions by default run on -[build machines provided by GitHub](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). -For Unity projects the available disk size is quite small. You may experience an error related to -running out of disk space. You may also want to run the build on a server with more memory or -processing resources. +## Self-Hosted Runners vs Orchestrator -### GitHub Self-Hosted Runners vs Game CI Orchestrator +Both options let you build larger projects. Here's when to pick which: -_GitHub users can consider: -[GitHub self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) -and Orchestrator. Both can enable you to build larger projects._ - -_Orchestrator is better if you don't have a server setup or don't want to manage or maintain your -own build server._ - -_Self-hosted runners are best used when you already have a server available, running 24/7 that you -can setup as a runner. And you're happy to maintain and keep that server available and running._ +| | Self-Hosted Runners | Orchestrator | +| --------------- | ---------------------------------- | ------------------------------------- | +| **Setup** | Requires a dedicated server | Cloud account + credentials | +| **Maintenance** | You manage the server 24/7 | Fully managed, no servers to maintain | +| **Cost model** | Fixed (server always running) | Pay per build (scales to zero) | +| **Best for** | Teams with existing infrastructure | Teams without dedicated servers | diff --git a/docs/03-github-orchestrator/03-examples/01-command-line.mdx b/docs/03-github-orchestrator/03-examples/01-command-line.mdx index 636144e9..17b7f4ec 100644 --- a/docs/03-github-orchestrator/03-examples/01-command-line.mdx +++ b/docs/03-github-orchestrator/03-examples/01-command-line.mdx @@ -3,47 +3,67 @@ _Preview Support Only_ You can install Game CI locally and start orchestrator jobs from the command line or by integrating -your own tools. All parameters in [API Reference](../api-reference) can be specified as command line -input fields. +your own tools. All parameters in the [API Reference](../api-reference) can be specified as CLI +flags. -# Install - -Currently (development) +## Install ```bash git clone https://github.com/game-ci/unity-builder.git +cd unity-builder yarn install -yarn run cli -m {mode parameter} --projectPath {Your project path} {... other command line parameters} ``` -# Planned (does not work currently) +## Usage + +```bash +yarn run cli -m --projectPath [options] +``` + +### Examples -We plan to offer support for Game CI via Deno. This will enable fast, typescript native runtime and -you will be able to access this via the following: +Run a standard build: ```bash -dpx game-ci build +yarn run cli -m cli-build \ + --projectPath /path/to/your/unity/project \ + --providerStrategy aws \ + --targetPlatform StandaloneLinux64 \ + --gitPrivateToken $GIT_TOKEN ``` -# Help +List active resources: + +```bash +yarn run cli -m list-resources --providerStrategy aws +``` -_You can run `yarn run cli -h` or `yarn run cli --help` to list all modes and paramters with -descriptions_ +Watch a running build: + +```bash +yarn run cli -m watch --providerStrategy aws +``` + +Clean up old resources: + +```bash +yarn run cli -m garbage-collect --providerStrategy aws +``` -# Main Command Parameters +## Help -- Default: `cli` (runs a standard build workflow) -- See API Reference "Modes" +Run `yarn run cli --help` to list all modes and parameters with descriptions. -# Keeping command line parameters short +## Keeping Commands Short -You can avoid specifying long command line input for credentials by using environment variables or -[the input override feature](../advanced-topics/configuration-override#example) to shorten commands -signficantly. +You can avoid long CLI flags for credentials by using environment variables or the +[Configuration Override](../advanced-topics/configuration-override#example) feature. -This enables you to provide a command to pull input, e.g you can pull from a file or from a secret -manager. +This lets you pull input from a file or secret manager: ```bash -yarn run cli --populateOverride true --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD --inputPullCommand="gcloud secrets versions access 1 --secret=\"{0}\"" +yarn run cli \ + --populateOverride true \ + --pullInputList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD \ + --inputPullCommand='gcloud secrets versions access 1 --secret="{0}"' ``` diff --git a/docs/03-github-orchestrator/03-examples/02-github-examples/02-aws.mdx b/docs/03-github-orchestrator/03-examples/02-github-examples/02-aws.mdx index 3c18f3e3..442dad1e 100644 --- a/docs/03-github-orchestrator/03-examples/02-github-examples/02-aws.mdx +++ b/docs/03-github-orchestrator/03-examples/02-github-examples/02-aws.mdx @@ -2,18 +2,20 @@ ## Requirements -- You must have an AWS account setup and ready to create resources. -- Create a service account and generate an AWS access key and key id. +- An AWS account with permission to create resources (ECS, CloudFormation, S3, Kinesis, CloudWatch). +- An IAM user or role with an access key and secret key. ## AWS Credentials -Setup the following as `env` variables for orchestrator to use: +Set the following as `env` variables in your workflow: -- `AWS_ACCESS_KEY_ID` -- `AWS_SECRET_ACCESS_KEY` -- `AWS_DEFAULT_REGION` (should be the same AWS region as the base stack e.g `eu-west-2`) +| Variable | Description | +| ----------------------- | ------------------------------------------------------- | +| `AWS_ACCESS_KEY_ID` | IAM access key ID. | +| `AWS_SECRET_ACCESS_KEY` | IAM secret access key. | +| `AWS_DEFAULT_REGION` | AWS region matching your base stack (e.g. `eu-west-2`). | -If you're using GitHub you can use a GitHub Action: +If you're using GitHub Actions, configure credentials with: ```yaml - name: Configure AWS Credentials @@ -24,48 +26,49 @@ If you're using GitHub you can use a GitHub Action: aws-region: eu-west-2 ``` -_Note:_ _This enables Orchestrator access AWS._ +## CPU and Memory -## Configuration For AWS Orchestrator Jobs +AWS Fargate only accepts specific CPU/memory combinations. Values use the format `1024 = 1 vCPU` or +`1 GB`. Do not include the vCPU or GB suffix. -Refer to [Configuration page](../../api-reference) or the [example below](#example). +See the full list: +[AWS Fargate Task Definitions](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size) -### Allowed CPU/Memory Combinations +Common combinations: -There are some limitations to the CPU and Memory parameters. AWS will only accept the following -combinations: -[AWS Fargate Documentation, Allowed CPU and memory values (Task Definitions)](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size) +| CPU (`containerCpu`) | Memory (`containerMemory`) | +| -------------------- | -------------------------- | +| `256` (0.25 vCPU) | `512`, `1024`, `2048` | +| `512` (0.5 vCPU) | `1024` – `4096` | +| `1024` (1 vCPU) | `2048` – `8192` | +| `2048` (2 vCPU) | `4096` – `16384` | +| `4096` (4 vCPU) | `8192` – `30720` | -#### Summary Of Format - -- Values are represented as 1024:1 GB or CPU. -- Do not include the vCPU or GB suffix. -- 1 CPU can go to a max of 6 GB of memory. 2 CPU's are required to go higher. - -#### Valid CPU and Memory Values - -```yaml -- orchestratorMemory: 4096 -- orchestratorCpu: 1024 -``` - -## Example +## Example Workflow ```yaml -- uses: game-ci/unity-builder@orchestrator-develop +- uses: game-ci/unity-builder@v4 id: aws-fargate-unity-build with: providerStrategy: aws versioning: None - projectPath: `your path here` - unityVersion: `unity version here` + projectPath: path/to/your/project + unityVersion: 2022.3.0f1 targetPlatform: ${{ matrix.targetPlatform }} gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} - # You may want to export your builds somewhere external so you can access it: + containerCpu: 1024 + containerMemory: 4096 + # Export builds to S3: containerHookFiles: aws-s3-upload-build ``` -_[Custom Steps](../../advanced-topics/custom-hooks/container-hooks)_ +See [Container Hooks](../../advanced-topics/custom-hooks/container-hooks) for more on +`containerHookFiles`. + +A full workflow example is available in the builder source: +[orchestrator-pipeline.yml](https://github.com/game-ci/unity-builder/blob/main/.github/workflows/orchestrator-pipeline.yml). + +## AWS Parameters -A full workflow example can be seen in builder's -[Orchestrator GitHub sourcecode for GitHub Pipeline](https://github.com/game-ci/unity-builder/blob/309d668d637ae3e7ffe90d61612968db92e1e376/.github/workflows/orchestrator-pipeline.yml#L109). +For the full list of AWS-specific parameters (`awsStackName`, endpoint overrides, etc.), see the +[API Reference — AWS section](../../api-reference#aws). diff --git a/docs/03-github-orchestrator/03-examples/02-github-examples/03-kubernetes.mdx b/docs/03-github-orchestrator/03-examples/02-github-examples/03-kubernetes.mdx index a71f41a8..f5b5faf1 100644 --- a/docs/03-github-orchestrator/03-examples/02-github-examples/03-kubernetes.mdx +++ b/docs/03-github-orchestrator/03-examples/02-github-examples/03-kubernetes.mdx @@ -2,50 +2,64 @@ ## Requirements -- You must have a Kubernetes cluster setup and ready that supports persistent volumes. -- Create a kubeconfig and encode it as base64. +- A running Kubernetes cluster that supports persistent volumes. +- A kubeconfig file encoded as base64. ## K8s Credentials -Setup the following as `env` variables for the GitHub build step: +Pass the base64-encoded kubeconfig via the `kubeConfig` parameter or as an environment variable: -- `kubeConfig` (should be encoded as base64) - -## Configuration For Kubernetes Orchestrator Jobs - -Refer to [Configuration page](../../api-reference) or the [example below](#example). - -### Allowed CPU/Memory Combinations - -- `0.25 vCPU` - 0.5 GB, 1 GB, 2 GB -- `0.5 vCPU` - 1 GB, 2 GB, 3 GB, 4 GB -- `1 vCPU` - 2 GB, 3 GB, 4 GB, 5 GB, 6 GB, 7 GB, 8 GB -- `2 vCPU` - Between 4 GB and 16 GB in 1-GB increments -- `4 vCPU` - Between 8 GB and 30 GB in 1-GB increments +```yaml +env: + kubeConfig: ${{ secrets.KUBE_CONFIG }} +``` -#### Summary Of Format +## CPU and Memory -- Values are represented as 1024:1 GB or CPU. +Kubernetes accepts the same unit format as AWS — `1024 = 1 vCPU`, memory in MB. Do not include the +vCPU or GB suffix. -Do not include the vCPU or GB suffix. +| CPU (`containerCpu`) | Memory (`containerMemory`) | +| -------------------- | -------------------------- | +| `256` (0.25 vCPU) | `512`, `1024`, `2048` | +| `512` (0.5 vCPU) | `1024` – `4096` | +| `1024` (1 vCPU) | `2048` – `8192` | +| `2048` (2 vCPU) | `4096` – `16384` | +| `4096` (4 vCPU) | `8192` – `30720` | -### Example +## Example Workflow ```yaml -- uses: game-ci/unity-builder@orchestrator-develop +- uses: game-ci/unity-builder@v4 id: k8s-unity-build with: providerStrategy: k8s versioning: None - projectPath: `your path here` - unityVersion: `unity version here` + projectPath: path/to/your/project + unityVersion: 2022.3.0f1 targetPlatform: ${{ matrix.targetPlatform }} gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} - # You may want to export your builds somewhere external so you can access it: + kubeVolumeSize: 10Gi + containerCpu: 1024 + containerMemory: 4096 + # Export builds to S3: containerHookFiles: aws-s3-upload-build ``` -_[Custom Steps](../../advanced-topics/custom-hooks/container-hooks)_ +See [Container Hooks](../../advanced-topics/custom-hooks/container-hooks) for more on +`containerHookFiles`. + +A full workflow example is available in the builder source: +[orchestrator-k8s-pipeline.yml](https://github.com/game-ci/unity-builder/blob/main/.github/workflows/orchestrator-k8s-pipeline.yml). + +### Cluster Tips + +- **Keep the cluster running.** Cold-starting a Kubernetes cluster is slow. If you need auto-scaling + to zero, consider Google Cloud Kubernetes Autopilot. +- **Cloud storage required.** Kubernetes support currently requires cloud storage for caching. S3 is + built-in, or use rclone for other backends. + +## K8s Parameters -A full workflow example can be seen in builder's -[Orchestrator GitHub sourcecode for AWS Pipeline](https://github.com/game-ci/unity-builder/blob/main/.github/workflows/orchestrator-k8s-pipeline.yml). +For the full list of Kubernetes parameters (`kubeConfig`, `kubeVolume`, `kubeStorageClass`, etc.), +see the [API Reference — Kubernetes section](../../api-reference#kubernetes). diff --git a/docs/03-github-orchestrator/04-api-reference.mdx b/docs/03-github-orchestrator/04-api-reference.mdx index 6b1b85d1..45b6a31e 100644 --- a/docs/03-github-orchestrator/04-api-reference.mdx +++ b/docs/03-github-orchestrator/04-api-reference.mdx @@ -1,260 +1,158 @@ # API Reference -## Configuration - -_You can specify input parameters via any of the following methods._ - -- **GitHub Action `with`** _See "Getting Started" examples._ -- **Command Line** _You can specify input parameters via command line._ -- **Environment Variables** _You can specify input parameters via environment variables._ -- **Configuration Override** _[Advanced Topics / Overrides](advanced-topics/configuration-override)_ - -## Modes - -Orchestrator can accept a parameter to run a specific mode, by default cli-build is run. - -```bash -cli-build -``` - -_runs an orchestrator build_ - -```bash -list-resources -``` - -_lists active resources_ - -```bash -list-workflow -``` - -_lists running workflows_ - -```bash -watch -``` - -_follows logs of a running workflow_ - -```bash -garbage-collect -``` - -_runs garbage collection_ - -```bash -- cache-push -- cache-pull -``` - -Cache commands to push and pull from the local caching directory. Used in orchestrator workflows. -Uses `cachePullFrom` and `cachePushTo` parameters. - -```bash -- hash (hash folder contents recursively) -- print-input (prints all input parameters) -``` - -Utility commands - -```bash -- remote-cli-pre-build (sets up a repository, usually before a game-ci build) -- remote-cli-post-build (pushes to LFS and Library cache) -``` - -Commands called during orchestrator workflows before/after a build. - -## Common Parameters - -### Git synchronization parameters - -```bash -gitPrivateToken (should be a GitHub access token with permission to get repositories) -``` - -Used to authenticate remote job's access to repository. Also used for LFS file pulling if -`GIT_PRIVATE_TOKEN` is not set separately. - -```bash -- GITHUB_REPOSITORY -- GITHUB_REF || branch || GitSHA -``` - -Used to synchronize the repository to the Orchestrator job. If parameters are not provided, will -attempt to read them from current directory's git repo (e.g branch, commit SHA, remote URL). - -### Orchestrator parameters - -```bash -providerStrategy -``` - -Specifies the Cloud Provider to use for Orchestrator jobs. Built-in values: `aws`, `k8s`, -`local-docker`, `local`. You can also specify a custom provider via GitHub URL, NPM package, or -local path (see [Custom Providers](advanced-topics/custom-providers)). - -```bash -- containerCpu -- containerMemory -``` - -Specifies the CPU and Memory resources to be used for cloud containers created by Orchestrator. -(See: getting started section for more configuration options per provider.) - -```bash -orchestratorBranch -``` - -Specifies the release branch of Orchestrator to use for remote containers. Accepted values: `main` -(default), `orchestrator-develop` (latest/development). - -```bash -cloneDepth -``` - -Specifies the depth of the git clone for the repository. Defaults to `50`. Use `0` for a full clone. - -### Custom commands from files parameters - -```bash -- containerHookFiles -- commandHookFiles -- commandHooks -- postBuildContainerHooks -- preBuildContainerHooks -``` - -Specifies the name of custom hook or step files to include in workflow. (Accepted Format: see -"[container hooks](advanced-topics/custom-hooks/container-hooks) -[command hooks](advanced-topics/custom-hooks/command-hooks)") - -### Custom commands from yaml parameters - -```bash -customJob -``` - -Specifies a custom job to override default build workflow. (Accepted Format: see -"[advanced topics / custom job](advanced-topics/custom-hooks/custom-job)") +## ⚙️ Configuration Methods + +| Method | Description | +| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| **GitHub Action `with`** | Pass parameters directly in your workflow file. See [Getting Started](getting-started). | +| **Command Line** | Pass parameters as CLI flags. See [Command Line](examples/command-line). | +| **Environment Variables** | Set parameters as environment variables in your shell or CI environment. | +| **Configuration Override** | Pull parameters dynamically from secret managers or files. See [Configuration Override](advanced-topics/configuration-override). | + +## 🔧 Modes + +Set the mode to control what Orchestrator does. Default: `cli-build`. + +| Mode | Description | +| ----------------------- | ------------------------------------------------------------------------------------- | +| `cli-build` | Run a standard build workflow. | +| `list-resources` | List active cloud resources. | +| `list-workflow` | List running workflows. | +| `watch` | Follow logs of a running workflow. | +| `garbage-collect` | Clean up old resources. See [Garbage Collection](advanced-topics/garbage-collection). | +| `cache-push` | Push to the caching directory. Uses `cachePushTo`. | +| `cache-pull` | Pull from the caching directory. Uses `cachePullFrom`. | +| `hash` | Hash folder contents recursively. | +| `print-input` | Print all resolved input parameters. | +| `remote-cli-pre-build` | Set up a repository before a build (used internally by workflows). | +| `remote-cli-post-build` | Push LFS files and Library cache after a build (used internally). | + +## 📋 Parameters + +### Provider + +| Parameter | Default | Description | +| ---------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `providerStrategy` | `local` | Cloud provider to use. Built-in: `aws`, `k8s`, `local-docker`, `local`. Also accepts a GitHub URL, NPM package, or local path for [custom providers](advanced-topics/providers/custom-providers). | +| `containerCpu` | `1024` | CPU units for cloud containers (`1024` = 1 vCPU). See provider setup guides for allowed values. | +| `containerMemory` | `3072` | Memory in MB for cloud containers (`4096` = 4 GB). See provider setup guides for allowed values. | +| `orchestratorBranch` | `main` | Release branch of Orchestrator for remote containers. Use `orchestrator-develop` for latest development builds. | +| `orchestratorRepoName` | `game-ci/unity-builder` | Repository for Orchestrator source. Override to use a fork for testing or custom builds. | + +### Git Synchronization + +| Parameter | Default | Description | +| ------------------- | -------- | ------------------------------------------------------------------- | +| `gitPrivateToken` | — | GitHub access token with repo access. Used for git clone and LFS. | +| `githubOwner` | — | GitHub owner or organization name. | +| `GITHUB_REPOSITORY` | _(auto)_ | Repository in `owner/repo` format. Auto-detected in GitHub Actions. | +| `GITHUB_REF` | _(auto)_ | Git ref to build. Falls back to `branch` or `GitSHA` parameters. | +| `cloneDepth` | `50` | Depth of the git clone. Use `0` for a full clone. | +| `allowDirtyBuild` | `false` | Allow building from a branch with uncommitted changes. | + +### Custom Hooks + +| Parameter | Default | Description | +| ------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------ | +| `containerHookFiles` | — | Names of [container hook](advanced-topics/custom-hooks/container-hooks) files from `.game-ci/container-hooks/`. | +| `customHookFiles` | — | Names of custom hook files from `.game-ci/hooks/`. | +| `customCommandHooks` | — | Inline [command hooks](advanced-topics/custom-hooks/command-hooks) as YAML. | +| `postBuildSteps` | — | Post-build job in YAML format with keys: `image`, `secrets`, `command`. | +| `preBuildSteps` | — | Pre-build job (after repo setup, before build) in YAML format. | +| `postBuildContainerHooks` | — | Container hook files to run after the build step. | +| `preBuildContainerHooks` | — | Container hook files to run before the build step. | +| `customJob` | — | Custom job YAML to override the default build workflow. See [Custom Job](advanced-topics/custom-hooks/custom-job). | ### Configuration Override -```bash -readInputOverrideCommand -``` - -Read parameter from command line output, such as a secret manager. Must include a `{0}` to inject -the name of the parameter to pull. Built-in presets: `gcp-secret-manager`, `aws-secret-manager`. -(See: [Configuration Override](advanced-topics/configuration-override)) - -```bash -readInputFromOverrideList -``` - -Comma separated list of parameters to apply with `readInputOverrideCommand`. (See: -[Configuration Override](advanced-topics/configuration-override)) +| Parameter | Default | Description | +| --------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `readInputOverrideCommand` | — | Command to read a parameter value from an external source. Use `{0}` as the parameter name placeholder. Built-in presets: `gcp-secret-manager`, `aws-secret-manager`. See [Configuration Override](advanced-topics/configuration-override). | +| `readInputFromOverrideList` | — | Comma-separated list of parameter names to pull via `readInputOverrideCommand`. | ### Storage -```bash -storageProvider -``` - -Specifies the storage backend for caching and artifacts. Accepted values: `s3` (default), `rclone`. - -```bash -rcloneRemote -``` - -Configures the rclone remote storage endpoint. Required when using `storageProvider: rclone`. +| Parameter | Default | Description | +| ----------------- | ------- | --------------------------------------------------------------------------- | +| `storageProvider` | `s3` | Storage backend for caching and artifacts. Accepted values: `s3`, `rclone`. | +| `rcloneRemote` | — | Rclone remote endpoint. Required when `storageProvider` is `rclone`. | ### AWS -```bash -awsStackName -``` - -Name of the persistent shared base stack, used to store artifacts and caching. Defaults to -`game-ci`. - -```bash -- awsEndpoint (base endpoint override for all AWS services) -- awsCloudFormationEndpoint -- awsEcsEndpoint -- awsKinesisEndpoint -- awsCloudWatchLogsEndpoint -- awsS3Endpoint -``` - -Optional AWS service endpoint overrides. Useful for testing with LocalStack or other AWS-compatible -services. - -### K8s - -```bash -- kubeConfig (base64 encoded kubernetes config) -- kubeVolume -- kubeVolumeSize (default: 5Gi) -- kubeStorageClass -``` - -Override name of persistent volume used, size of volume and storage class used. +| Parameter | Default | Description | +| --------------------------- | --------- | -------------------------------------------------------------- | +| `awsStackName` | `game-ci` | Name of the persistent shared CloudFormation base stack. | +| `awsEndpoint` | — | Base endpoint override for all AWS services (e.g. LocalStack). | +| `awsCloudFormationEndpoint` | — | CloudFormation service endpoint override. | +| `awsEcsEndpoint` | — | ECS service endpoint override. | +| `awsKinesisEndpoint` | — | Kinesis service endpoint override. | +| `awsCloudWatchLogsEndpoint` | — | CloudWatch Logs service endpoint override. | +| `awsS3Endpoint` | — | S3 service endpoint override. | + +### Kubernetes + +| Parameter | Default | Description | +| ------------------ | ------- | ----------------------------------------------------------------------- | +| `kubeConfig` | — | Base64-encoded Kubernetes config file. | +| `kubeVolume` | — | Name of the persistent volume claim to use. | +| `kubeVolumeSize` | `5Gi` | Size of the persistent volume. | +| `kubeStorageClass` | — | Storage class for the persistent volume. Empty = auto-install via rook. | ### Caching -```bash -cacheKey -``` +| Parameter | Default | Description | +| ----------------------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `cacheKey` | _(branch name)_ | Scope for sharing cache entries. Builds with the same key share a cache. | +| `maxRetainedWorkspaces` | `0` | Maximum number of [retained workspaces](advanced-topics/retained-workspace). `0` = unlimited. Above the limit, jobs use standard caching. | -Defaults to branch name. Defines the scope for sharing cache entries. +### GitHub Integration -### Utility +| Parameter | Default | Description | +| ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------- | +| `githubCheck` | `false` | Create a GitHub Check for each orchestrator step. See [GitHub Integration](advanced-topics/providers/github-integration). | +| `asyncOrchestrator` | `false` | Run in async mode — returns immediately without waiting for the build to complete. | +| `watchToEnd` | `true` | Whether to follow the build logs until completion. | -```bash -- orchestratorDebug (Debug logging for Orchestrator) -- resourceTracking (Enable resource tracking logs for disk usage summaries) -- useLargePackages (Any packages in manifest.json containing phrase "LargePackage" will be - redirected to a shared folder for all builds sharing a cache key) -- useSharedBuilder (Use a shared clone of Game-CI, saves some storage space and can be used if - you're using one release branch of Orchestrator) -- useCompressionStrategy (Use Lz4 compression for cache and build artifacts. Enabled by default) -- watchToEnd (Whether to watch the build to the end, default: true) -- asyncOrchestrator (Run in async mode, returns immediately without waiting for build completion) -``` +### Build Options -### Retained Workspace - -```bash -- maxRetainedWorkspaces -``` - -See: [Advanced Topics / Retained Workspaces](advanced-topics/retained-workspace), enables caching -entire project folder. +| Parameter | Default | Description | +| ------------------------ | ------- | ------------------------------------------------------------------------------------------------------------------------ | +| `orchestratorDebug` | `false` | Enable verbose debug logging (resource tracking, directory tree, environment listing). | +| `resourceTracking` | `false` | Enable resource tracking logs with disk usage summaries. | +| `useLargePackages` | `false` | Redirect packages containing "LargePackage" in `manifest.json` to a shared folder across builds with the same cache key. | +| `useSharedBuilder` | `false` | Use a shared clone of Game CI. Saves storage when using a single Orchestrator release branch. | +| `useCompressionStrategy` | `true` | Use LZ4 compression for cache and build artifacts. | +| `useCleanupCron` | `true` | Create an AWS CloudFormation cron job to automatically clean up old resources. | ### Garbage Collection -```bash -- garbageMaxAge (Maximum age in hours before resources are cleaned up, default: 24) -``` +| Parameter | Default | Description | +| --------------- | ------- | ----------------------------------------------------- | +| `garbageMaxAge` | `24` | Maximum age in hours before resources are cleaned up. | + +## 🖥️ CLI-Only Parameters -## Command Line Only Parameters +These parameters are only available when using Orchestrator from the command line. -```bash -- populateOverride -- cachePushFrom -- cachePushTo -- artifactName -- select -``` +| Parameter | Description | +| ------------------ | ---------------------------------------------------- | +| `populateOverride` | Enable reading parameters from the override command. | +| `cachePushFrom` | Local directory to push cache from. | +| `cachePushTo` | Remote path to push cache to. | +| `artifactName` | Name for the build artifact. | +| `select` | Select a specific workflow or resource by name. | -## Other Environment Variables +## 🌍 Environment Variables -```bash -- USE_IL2CPP (Set to `false`) -``` +| Variable | Description | +| ---------------------------------- | --------------------------------------------------------------------------------- | +| `USE_IL2CPP` | Set to `false` to disable IL2CPP builds. | +| `AWS_FORCE_PROVIDER` | Force provider when LocalStack is detected. Values: `aws`, `aws-local`, or empty. | +| `ORCHESTRATOR_AWS_STACK_WAIT_TIME` | CloudFormation stack timeout in seconds. Default: `600`. | +| `PURGE_REMOTE_BUILDER_CACHE` | Set to clear the remote builder cache before a build. | +| `GIT_PRIVATE_TOKEN` | Separate token for LFS pulls (falls back to `gitPrivateToken`). | -# External Links +## 🔗 External Links -All accepted parameters given here with a description: -[https://github.com/game-ci/unity-builder/blob/main/action.yml](https://github.com/game-ci/unity-builder/blob/main/action.yml) +All parameters with descriptions: +[game-ci/unity-builder action.yml](https://github.com/game-ci/unity-builder/blob/main/action.yml) diff --git a/docs/03-github-orchestrator/06-advanced-topics/01-caching.mdx b/docs/03-github-orchestrator/06-advanced-topics/01-caching.mdx index 7e04489b..a7eb3563 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/01-caching.mdx +++ b/docs/03-github-orchestrator/06-advanced-topics/01-caching.mdx @@ -1,42 +1,54 @@ # Caching -Orchestrator supports two main caching modes: +Orchestrator supports two caching strategies. You can mix both by setting `maxRetainedWorkspaces` — +once the limit is reached, additional jobs fall back to standard caching. + +``` + Standard Caching Retained Workspace + ┌─────────────────┐ ┌─────────────────┐ + │ 📁 LFS files │ │ 📁 Entire │ + │ 📁 Library/ │ │ project │ + │ │ │ folder │ + │ Smaller storage │ │ Faster builds │ + │ Good for small │ │ Good for large │ + │ projects │ │ projects │ + └─────────────────┘ └─────────────────┘ +``` -- Standard Caching -- Retained Workspace - -_You can even mix the two by specifying a "MaxRetainedWorkspaces" parameter. Above the max_ -_concurrent jobs a new job will use standard caching._ - -## Storage Providers - -Orchestrator supports two storage backends for caching: +## Standard Caching -- **S3** (default) - Uses AWS S3 for cache storage. Works with both AWS and LocalStack. -- **Rclone** - Uses rclone for flexible cloud storage. Supports many storage backends. +Caches only the Unity **Library** folder and **LFS** files between builds. Uses less storage but +requires re-importing unchanged assets. -Configure via the `storageProvider` parameter. When using rclone, also set `rcloneRemote` to your -configured remote endpoint. +- ✅ Minimum storage cost +- ✅ Best for smaller projects +- ⚠️ Slower rebuilds for large asset libraries -## Standard Caching +## Retained Workspace -#### Good For +Caches the **entire project folder** between builds. Provides the fastest rebuilds but consumes more +storage. -- Minimum storage use -- Smaller projects +- ✅ Maximum build speed +- ✅ Best for large projects with long import times +- ⚠️ Higher storage cost -#### What is cached between builds +See [Retained Workspaces](retained-workspace) for configuration details. -- LFS files -- Unity Library folder +## 🗄️ Storage Providers -## Retained Workspace +| Provider | `storageProvider` | Description | +| -------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| S3 | `s3` (default) | AWS S3 storage. Works with both AWS and LocalStack. | +| Rclone | `rclone` | Flexible cloud storage via [rclone](https://rclone.org). Supports 70+ backends (Google Cloud, Azure Blob, Backblaze, SFTP, etc). | -#### Good For +Configure with the `storageProvider` parameter. When using rclone, also set `rcloneRemote` to your +configured remote endpoint. -- Maximum build speed -- Larger projects with long import times +## 🔒 Workspace Locking -#### What is cached between builds +When using retained workspaces, Orchestrator uses distributed locking (via S3 or rclone) to ensure +only one build uses a workspace at a time. This enables safe concurrent builds that share and reuse +workspaces without conflicts. -- Entire Project Folder +Locking is managed automatically — no configuration required beyond setting `maxRetainedWorkspaces`. diff --git a/docs/03-github-orchestrator/06-advanced-topics/02-retained-workspace.mdx b/docs/03-github-orchestrator/06-advanced-topics/02-retained-workspace.mdx index 0e1bfbcd..cce787cf 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/02-retained-workspace.mdx +++ b/docs/03-github-orchestrator/06-advanced-topics/02-retained-workspace.mdx @@ -1,8 +1,32 @@ # Retained Workspaces -Caches the entire project folder. This option provides the best responsiveness, but also can consume -lots of storage. +Retained workspaces cache the **entire project folder** between builds. This provides the fastest +possible rebuilds at the cost of more cloud storage. -Using API: `maxRetainedWorkspaces`. You can specify a maximum number of retained workspaces, only -one job can use a retained workspace at one time. Each retained workspace consumes more cloud -storage. +## Configuration + +Set `maxRetainedWorkspaces` to control how many workspaces are kept: + +| Value | Behavior | +| ----- | ------------------------------------------------------------------------- | +| `0` | Unlimited retained workspaces (default). | +| `> 0` | Keep at most N workspaces. Additional jobs fall back to standard caching. | + +Each retained workspace is locked during use — only one build can use a workspace at a time. +Orchestrator handles locking automatically via S3 or rclone. + +## Example + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + maxRetainedWorkspaces: 3 + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +## ⚠️ Storage Considerations + +Each retained workspace stores a full copy of your project. For a 20 GB project with 3 retained +workspaces, expect ~60 GB of cloud storage usage. diff --git a/docs/03-github-orchestrator/06-advanced-topics/03-garbage-collection.mdx b/docs/03-github-orchestrator/06-advanced-topics/03-garbage-collection.mdx index b49d1a52..6400f1f1 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/03-garbage-collection.mdx +++ b/docs/03-github-orchestrator/06-advanced-topics/03-garbage-collection.mdx @@ -1,29 +1,16 @@ # Garbage Collection -Orchestrator creates, manages and destroys cloud workloads you request. Resources have to be -created. +Orchestrator creates cloud resources (containers, stacks, volumes) for each build and cleans them up +automatically. If a build fails or is interrupted, resources may be left behind. -It is possible a resource doesn't get deleted by orchestrator after a failed or interrupted build. - -You can use garbage collection to verify everything has been cleaned up. - -Use the **Mode**: `garbage-collect`. - -## Parameters - -```bash -garbageMaxAge -``` - -Maximum age in hours before resources are considered stale and eligible for cleanup. Defaults to -`24`. +Use garbage collection to clean up stale resources. ## Usage ### GitHub Actions ```yaml -- uses: game-ci/unity-builder@main +- uses: game-ci/unity-builder@v4 with: providerStrategy: aws mode: garbage-collect @@ -35,3 +22,15 @@ Maximum age in hours before resources are considered stale and eligible for clea ```bash yarn run cli -m garbage-collect --providerStrategy aws ``` + +## Parameters + +| Parameter | Default | Description | +| --------------- | ------- | ----------------------------------------------------- | +| `garbageMaxAge` | `24` | Maximum age in hours before resources are cleaned up. | + +## 🔄 Automatic Cleanup + +When using the AWS provider, Orchestrator can create a CloudFormation-based cleanup cron job that +automatically removes old ECS task definitions and resources. This is controlled by the +`useCleanupCron` parameter (enabled by default). diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-configuration-override.mdx b/docs/03-github-orchestrator/06-advanced-topics/04-configuration-override.mdx index 94134fc1..e2ac2640 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/04-configuration-override.mdx +++ b/docs/03-github-orchestrator/06-advanced-topics/04-configuration-override.mdx @@ -1,19 +1,42 @@ # Configuration Override -When running any unity workload you must provide valid unity credentials. In addition to any other -credentials this is already quite a lot of input. For this reason, it is common to use the command -line mode with input override (link here). This enables you to provide a command to pull input, with -this approach you can create a file to store credentials or pull from a secret manager. +Pull parameter values from external sources like secret managers or files at runtime. This avoids +hardcoding credentials and keeps CLI commands short. + +``` + Orchestrator External Source + ┌──────────────┐ command ┌──────────────────┐ + │ Reads input │ ──────────────► │ Secret Manager │ + │ override │ │ (GCP, AWS, file) │ + │ list │ ◄────────────── │ │ + │ │ value │ │ + └──────────────┘ └──────────────────┘ +``` + +## Parameters + +| Parameter | Description | +| --------------------------- | ------------------------------------------------------------------ | +| `populateOverride` | Must be `true` to enable override (CLI only). | +| `readInputFromOverrideList` | Comma-separated list of parameter names to pull. | +| `readInputOverrideCommand` | Command to run. Use `{0}` as a placeholder for the parameter name. | + +### Built-in Presets + +Instead of writing a full command, you can use these presets as the `readInputOverrideCommand`: + +| Preset | Expands to | +| -------------------- | ----------------------------------------------------- | +| `gcp-secret-manager` | `gcloud secrets versions access 1 --secret="{0}"` | +| `aws-secret-manager` | `aws secretsmanager get-secret-value --secret-id {0}` | ## Example ```bash -game-ci -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD --readInputOverrideCommand="gcloud secrets versions access 1 --secret=\"{0}\"" +yarn run cli -m cli-build \ + --populateOverride true \ + --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD \ + --readInputOverrideCommand='gcloud secrets versions access 1 --secret="{0}"' ``` -## Required Parameters - -- `populateOverride` - Must be true to run the commands. -- `readInputFromOverrideList` - Comma separated list of parameters to read from override command. -- `readInputOverrideCommand` - A command line command to run (The command is formatted to replace - "{0}" with the parameter parameter name). +This runs the GCP command for each parameter name in the list and uses the output as the value. diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/01-custom-job.mdx b/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/01-custom-job.mdx index ade52d59..22d35a90 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/01-custom-job.mdx +++ b/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/01-custom-job.mdx @@ -1,4 +1,19 @@ # Custom Jobs -You can run a custom job instead of the default build workflow simplfy by specifying the `customJob` -parameter. +Override the default build workflow entirely by specifying the `customJob` parameter with a YAML job +definition. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + customJob: | + - name: my-custom-step + image: ubuntu + commands: | + echo "Running custom job" + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +This replaces the standard build steps with your own. Useful for running non-Unity workloads or +fully custom pipelines through Orchestrator's cloud infrastructure. diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/03-command-hooks.mdx b/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/03-command-hooks.mdx index ee1f2378..8a4327e7 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/03-command-hooks.mdx +++ b/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/03-command-hooks.mdx @@ -1,6 +1,8 @@ # Command Hooks -Custom commands can be injected into the standard build workflow via yaml or files. +Inject custom shell commands into the standard build workflow at specific points. + +## Format ```yaml - name: example hook @@ -10,3 +12,26 @@ Custom commands can be injected into the standard build workflow via yaml or fil commands: | echo "hello world!" ``` + +## Hook Points + +| Step | When it runs | +| -------- | ------------------------------- | +| `before` | Before the build step starts. | +| `after` | After the build step completes. | + +## Usage + +Pass hooks inline via the `commandHooks` parameter or reference files from the `.game-ci/hooks/` +directory via `customHookFiles`. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + customHookFiles: my-hook + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +Place hook files at `.game-ci/hooks/my-hook.yaml` in your repository. diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/04-container-hooks.mdx b/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/04-container-hooks.mdx index 6e0a9adc..70bcdeea 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/04-container-hooks.mdx +++ b/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/04-container-hooks.mdx @@ -1,9 +1,9 @@ # Container Hooks -Custom docker container steps can be run as part of the standard build workflow. Custom steps can -also be run standalone. +Run custom Docker containers as steps in the build workflow. Useful for uploading artifacts, +deploying builds, or running additional tools. -Custom steps can be specified via the CustomSteps parameter or via Custom Step files. +## Format ```yaml - name: upload @@ -11,3 +11,22 @@ Custom steps can be specified via the CustomSteps parameter or via Custom Step f commands: | echo "hello world!" ``` + +## Usage + +Define container hooks inline via `preBuildContainerHooks` / `postBuildContainerHooks`, or reference +files from `.game-ci/container-hooks/` via `containerHookFiles`. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + containerHookFiles: aws-s3-upload-build + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +## Pre-built Hooks + +Orchestrator ships with ready-to-use hooks for S3, rclone, and Steam. See +[Premade Container Hooks](premade-container-jobs) for the full list. diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/05-premade-container-jobs.mdx b/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/05-premade-container-jobs.mdx index a87c9586..48a61d01 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/05-premade-container-jobs.mdx +++ b/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/05-premade-container-jobs.mdx @@ -1,38 +1,46 @@ -# Premade Container Jobs +# Premade Container Hooks -## Cache syncronization +Orchestrator ships with pre-built container hooks for common tasks. Use them by name with the +`containerHookFiles` parameter. -### Upload Cache entry to AWS S3 +```yaml +containerHookFiles: aws-s3-upload-build +``` -Upload cache results from build to AWS S3. +## 📦 AWS S3 -`aws-s3-upload-cache` +| Hook Name | Description | +| --------------------- | ----------------------------------------- | +| `aws-s3-upload-build` | Upload build artifacts to S3. | +| `aws-s3-pull-build` | Pull cached build artifacts from S3. | +| `aws-s3-upload-cache` | Upload Unity Library and LFS cache to S3. | +| `aws-s3-pull-cache` | Pull Unity Library and LFS cache from S3. | -### Download Latest Cache entry from AWS S3 +Requires AWS credentials configured. Respects `useCompressionStrategy` for LZ4 compression. -Downloads library and git LFS cache from AWS S3. +## 📂 Rclone -`aws-s3-pull-cache` +| Hook Name | Description | +| --------------------- | ---------------------------------------------- | +| `rclone-upload-build` | Upload build artifacts via rclone. | +| `rclone-pull-build` | Pull cached build artifacts via rclone. | +| `rclone-upload-cache` | Upload Unity Library and LFS cache via rclone. | +| `rclone-pull-cache` | Pull Unity Library and LFS cache via rclone. | -## Artifacts +Requires `storageProvider: rclone` and `rcloneRemote` to be configured. Uses the `rclone/rclone` +Docker image. Respects `useCompressionStrategy` for LZ4 compression. -## Upload Build Artifact To AWS S3 +## 🎮 Steam -`aws-s3-upload-build` +| Hook Name | Description | +| ---------------------- | --------------------------------------------- | +| `steam-deploy-client` | Deploy a client build to Steam via SteamCMD. | +| `steam-deploy-project` | Deploy a project build to Steam via SteamCMD. | -Upload build artifact to AWS S3. (Currently only supports lz4 enabled which is default.) +Uses the `steamcmd/steamcmd` Docker image. Requires the following secrets to be configured: -## Upload Project Artifact To AWS S3 (To Do) - -Upload project artifact to AWS S3. (Currently only supports lz4 enabled which is default.) - -## Artifact entire project folder (To Do) - -archive to tar format, compress with lz4 if enabled and store in persistent cache folder. (Can then -upload.) - -## Deploy - -## Upload to Steam (To Do) - -upload build folder to given steam branch +- `STEAM_USERNAME`, `STEAM_PASSWORD` +- `STEAM_APPID` +- `STEAM_SSFN_FILE_NAME`, `STEAM_SSFN_FILE_CONTENTS` +- `STEAM_CONFIG_VDF_1` through `STEAM_CONFIG_VDF_4` +- `BUILD_GUID_TARGET`, `RELEASE_BRANCH` diff --git a/docs/03-github-orchestrator/06-advanced-topics/05-logging.mdx b/docs/03-github-orchestrator/06-advanced-topics/05-logging.mdx index 99660d31..c044820d 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/05-logging.mdx +++ b/docs/03-github-orchestrator/06-advanced-topics/05-logging.mdx @@ -1,13 +1,34 @@ # Logging -Logs are streamed from the workload to the GameCI origin unless you use the +Orchestrator streams logs from the remote build back to your CI runner in real time. -## Kubernetes +``` + Cloud Container Orchestrator Your CI + ┌───────────────┐ ┌──────────┐ ┌──────────────┐ + │ Build output │ ─────► │ Log │ ─────► │ Console │ + │ │ │ stream │ │ output │ + └───────────────┘ └──────────┘ └──────────────┘ +``` -- Native Kubernetes logging api +## Provider-Specific Log Transport -## AWS +### Kubernetes -- Fargate log to Cloud Watch -- Subscription from Cloud Watch to Kinesis -- GameCI streams from logs from Kinesis +Uses the native Kubernetes logging API to stream pod logs directly. + +### AWS + +Logs flow through a CloudWatch → Kinesis pipeline: + +1. Fargate tasks write to **CloudWatch Logs** +2. A **Kinesis** subscription forwards logs in real time +3. Orchestrator consumes from the Kinesis stream + +## 🐛 Debug Logging + +Enable `orchestratorDebug: true` to get verbose output including: + +- Resource allocation summaries (CPU, memory, disk) +- Directory structure via `tree` +- Environment variable listing +- Disk usage snapshots (`df -h`, `du -sh`) diff --git a/docs/03-github-orchestrator/06-advanced-topics/06-secrets.mdx b/docs/03-github-orchestrator/06-advanced-topics/06-secrets.mdx index 17bae040..2c04fbb5 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/06-secrets.mdx +++ b/docs/03-github-orchestrator/06-advanced-topics/06-secrets.mdx @@ -1,14 +1,19 @@ # Secrets -Secrets are transferred to workload containers as secrets via the built-in secrets system the -provider being used supports. +Orchestrator securely transfers secrets to remote build containers using each provider's native +secrets system. ## Kubernetes -Secrets are created as native Kubernetes secret objects and mounted to job containers as env -variables. +Secrets are created as native **Kubernetes Secret** objects and mounted as environment variables in +job containers. ## AWS -Secrets are created as aws secret manager secrets and mounted to fargate tasks from secrets to env +Secrets are stored in **AWS Secrets Manager** and injected into Fargate tasks as environment variables. + +## 🔐 Managing Secrets + +For pulling secrets from external sources (e.g. GCP Secret Manager, AWS Secrets Manager) into +Orchestrator parameters, see [Configuration Override](configuration-override). diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/01-overview.mdx b/docs/03-github-orchestrator/06-advanced-topics/07-providers/01-overview.mdx new file mode 100644 index 00000000..217c17bf --- /dev/null +++ b/docs/03-github-orchestrator/06-advanced-topics/07-providers/01-overview.mdx @@ -0,0 +1,57 @@ +# Providers + +A **provider** is the backend that Orchestrator uses to run your builds. You choose a provider by +setting the `providerStrategy` parameter. + +## Built-in Providers + +These providers ship with Orchestrator and are maintained by the Game CI team. + +| Provider | `providerStrategy` | Description | Release Status | +| -------------- | ------------------ | ----------------------------------------------------------------------------- | -------------- | +| AWS Fargate | `aws` | Runs jobs on AWS Fargate (ECS). Fully managed, no servers to maintain. | Full release | +| Kubernetes | `k8s` | Runs jobs on any Kubernetes cluster. Flexible but requires a running cluster. | Preview | +| Local Docker | `local-docker` | Runs jobs in Docker containers on the local machine. | Preview | +| Local (direct) | `local` | Runs jobs directly on the local machine without containers. | Preview | + +### AWS Fargate + +Best for most users. Orchestrator automatically provisions and tears down Fargate tasks — you only +pay while a build is running. Requires an AWS account with IAM credentials. + +See [AWS setup guide](../../examples/github-examples/aws) for credentials and configuration. + +### Kubernetes + +Run builds on your own Kubernetes cluster. The cluster must support persistent volumes and should +stay running (cold starts are slow). Google Cloud Kubernetes Autopilot can scale to near-zero when +idle. + +See [Kubernetes setup guide](../../examples/github-examples/kubernetes) for credentials and +configuration. + +### Local Docker + +Runs the build workflow inside a Docker container on the machine running the action. Useful for +self-hosted runners with Docker installed. + +### Local + +Runs builds directly on the host machine with no container isolation. Useful for testing and +development. + +## Custom Providers + +You can extend Orchestrator with your own provider by pointing `providerStrategy` at a GitHub +repository, NPM package, or local file path. Orchestrator will dynamically load and validate it at +runtime. + +See [Custom Providers](custom-providers) for the full guide on creating and using custom providers. + +## Community Providers + +Community providers are third-party providers shared by the Game CI community. They are not +maintained by the Game CI team but have been reviewed and listed here for discoverability. + +See the [Community Providers](community-providers) page for the current list and instructions on how +to submit your own. diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-custom-providers.mdx b/docs/03-github-orchestrator/06-advanced-topics/07-providers/02-custom-providers.mdx similarity index 100% rename from docs/03-github-orchestrator/06-advanced-topics/07-custom-providers.mdx rename to docs/03-github-orchestrator/06-advanced-topics/07-providers/02-custom-providers.mdx diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/03-community-providers.mdx b/docs/03-github-orchestrator/06-advanced-topics/07-providers/03-community-providers.mdx new file mode 100644 index 00000000..213aaed9 --- /dev/null +++ b/docs/03-github-orchestrator/06-advanced-topics/07-providers/03-community-providers.mdx @@ -0,0 +1,47 @@ +# Community Providers + +Community providers are third-party Orchestrator providers built and shared by the community. They +are **not maintained by the Game CI team** but are listed here to help you discover and evaluate +options. + +:::caution + +Community providers are provided as-is. Review the source code and documentation of any community +provider before using it in your pipelines. + +::: + +## Provider List + +_No community providers have been submitted yet. Yours could be the first!_ + +{/\* When adding a provider, use this format: + +### Provider Name + +| | | +| -------------------- | -------------------------------------------------------------------------- | +| **Repository** | [user/repo](https://github.com/user/repo) | +| **providerStrategy** | `user/repo` | +| **Description** | Brief description of what the provider does and which platform it targets. | +| **Maintainer** | [@username](https://github.com/username) | + +\*/} + +## Submit Your Provider + +Built a custom provider? Share it with the community by adding it to this page. + +1. [Edit this file directly on GitHub](https://github.com/game-ci/documentation/edit/main/docs/03-github-orchestrator/06-advanced-topics/07-providers/03-community-providers.mdx) +2. Add your provider to the **Provider List** section above using the commented template +3. Submit a pull request for review + +Your provider should: + +- Have a public GitHub repository or published NPM package +- Implement the full [ProviderInterface](custom-providers#required-methods) +- Include a README with setup instructions +- Be actively maintained + +The Game CI team will review submissions for completeness before merging. Inclusion in this list +does not imply endorsement or a security guarantee. diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/04-github-integration.mdx b/docs/03-github-orchestrator/06-advanced-topics/07-providers/04-github-integration.mdx new file mode 100644 index 00000000..989573a6 --- /dev/null +++ b/docs/03-github-orchestrator/06-advanced-topics/07-providers/04-github-integration.mdx @@ -0,0 +1,41 @@ +# GitHub Integration + +Orchestrator has first-class support for GitHub Actions. When running from a GitHub Actions +workflow, Orchestrator automatically detects the repository, branch, and commit from the +environment. + +## GitHub Checks + +By enabling the `githubCheck` parameter, the orchestrator job will create a GitHub check for each +step. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + githubCheck: true + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +The step will show useful details about the job. This is especially useful in combination with async +mode, as you can run very long jobs and monitor their progress directly from the GitHub pull request +UI. + +## Async Mode + +Set `asyncOrchestrator: true` to start a build without waiting for it to complete. The GitHub Action +will return immediately and you can check progress via GitHub Checks or by running the `watch` mode. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + asyncOrchestrator: true + githubCheck: true + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +See the [AWS](../../examples/github-examples/aws) and +[Kubernetes](../../examples/github-examples/kubernetes) example pages for full workflow files. diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/05-gitlab-integration.mdx b/docs/03-github-orchestrator/06-advanced-topics/07-providers/05-gitlab-integration.mdx new file mode 100644 index 00000000..f7b59864 --- /dev/null +++ b/docs/03-github-orchestrator/06-advanced-topics/07-providers/05-gitlab-integration.mdx @@ -0,0 +1,30 @@ +# GitLab Integration + +You can use GitLab with Orchestrator via the Command Line mode. Orchestrator is not limited to +GitHub Actions — any CI system that can run shell commands can trigger orchestrator builds. + +## Setup + +1. Install the Orchestrator CLI (see [Command Line](../../examples/command-line)) +2. Set your git credentials and provider configuration as environment variables +3. Run the CLI from your `.gitlab-ci.yml` pipeline + +## Example `.gitlab-ci.yml` + +```yaml +build-unity: + stage: build + script: + - git clone https://github.com/game-ci/unity-builder.git /tmp/game-ci + - cd /tmp/game-ci && yarn install + - > + yarn run cli -m cli-build --projectPath $CI_PROJECT_DIR --providerStrategy aws + --gitPrivateToken $GIT_TOKEN + variables: + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + AWS_DEFAULT_REGION: eu-west-2 +``` + +See the [Command Line](../../examples/command-line) page for more details on CLI usage and the +[Configuration Override](../configuration-override) page for managing credentials. diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/_category_.yaml b/docs/03-github-orchestrator/06-advanced-topics/07-providers/_category_.yaml new file mode 100644 index 00000000..280c7133 --- /dev/null +++ b/docs/03-github-orchestrator/06-advanced-topics/07-providers/_category_.yaml @@ -0,0 +1,5 @@ +--- +position: 7.0 +label: Providers +collapsible: true +collapsed: true diff --git a/docs/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx b/docs/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx deleted file mode 100644 index 70e5079e..00000000 --- a/docs/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx +++ /dev/null @@ -1,3 +0,0 @@ -# GitLab Introduction - -You can use GitLab with Orchestrator via the Command Line mode. diff --git a/docs/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml b/docs/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml deleted file mode 100644 index 0407bcd5..00000000 --- a/docs/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -position: 9.0 -label: GitLab Integration -collapsible: true -collapsed: true diff --git a/docs/03-github-orchestrator/06-advanced-topics/10-github/01-github-checks.mdx b/docs/03-github-orchestrator/06-advanced-topics/10-github/01-github-checks.mdx deleted file mode 100644 index c0625fe3..00000000 --- a/docs/03-github-orchestrator/06-advanced-topics/10-github/01-github-checks.mdx +++ /dev/null @@ -1,7 +0,0 @@ -# GitHub Checks - -By enabling the `githubCheck` parameter, the orchestrator job will create a GitHub check for each -step. - -The step will show useful details about the job. This is especially useful in combination with async -mode, as you can run very long jobs. diff --git a/docs/03-github-orchestrator/06-advanced-topics/10-github/_category_.yaml b/docs/03-github-orchestrator/06-advanced-topics/10-github/_category_.yaml deleted file mode 100644 index ecbee03f..00000000 --- a/docs/03-github-orchestrator/06-advanced-topics/10-github/_category_.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -position: 10.0 -label: GitHub Integration -collapsible: true -collapsed: true From 172ae89333ac0eaa312b280445744ff9fc81fb59 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 05:53:19 +0000 Subject: [PATCH 03/20] Remove old standalone GitLab pages from versioned docs Content already merged into the providers section at 07-providers/05-gitlab-integration.mdx Co-Authored-By: Claude Opus 4.6 --- .../06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx | 3 --- .../06-advanced-topics/09-gitlab/_category_.yaml | 5 ----- .../06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx | 3 --- .../06-advanced-topics/09-gitlab/_category_.yaml | 5 ----- 4 files changed, 16 deletions(-) delete mode 100644 versioned_docs/version-2/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx delete mode 100644 versioned_docs/version-2/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml delete mode 100644 versioned_docs/version-3/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx delete mode 100644 versioned_docs/version-3/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml diff --git a/versioned_docs/version-2/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx b/versioned_docs/version-2/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx deleted file mode 100644 index 70e5079e..00000000 --- a/versioned_docs/version-2/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx +++ /dev/null @@ -1,3 +0,0 @@ -# GitLab Introduction - -You can use GitLab with Orchestrator via the Command Line mode. diff --git a/versioned_docs/version-2/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml b/versioned_docs/version-2/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml deleted file mode 100644 index 0407bcd5..00000000 --- a/versioned_docs/version-2/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -position: 9.0 -label: GitLab Integration -collapsible: true -collapsed: true diff --git a/versioned_docs/version-3/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx b/versioned_docs/version-3/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx deleted file mode 100644 index 70e5079e..00000000 --- a/versioned_docs/version-3/03-github-orchestrator/06-advanced-topics/09-gitlab/01-introduction-gitlab.mdx +++ /dev/null @@ -1,3 +0,0 @@ -# GitLab Introduction - -You can use GitLab with Orchestrator via the Command Line mode. diff --git a/versioned_docs/version-3/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml b/versioned_docs/version-3/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml deleted file mode 100644 index 0407bcd5..00000000 --- a/versioned_docs/version-3/03-github-orchestrator/06-advanced-topics/09-gitlab/_category_.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -position: 9.0 -label: GitLab Integration -collapsible: true -collapsed: true From c95128de19e780bf6aea8cf0d71e005641e16084 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:07:28 +0000 Subject: [PATCH 04/20] Restructure Orchestrator docs: promote Providers to top-level, improve cross-linking - Promote Providers from Advanced Topics to top-level section (05-providers/) with dedicated pages for AWS, Kubernetes, Local Docker, Local, Custom, Community, GitHub Integration, and GitLab Integration - Move Secrets out of Advanced Topics to top-level (06-secrets.mdx) - Rename custom-hooks to hooks throughout - Remove all WIP/preview/release-status notices (project is stable) - Fix floating {/* */} comment symbols in community-providers (use code block template) - Update ASCII diagram in Game-CI vs Orchestrator to show CLI/any CI dispatch - Add sidebar_label frontmatter for Game-CI vs Orchestrator page - Add comprehensive cross-linking across all orchestrator docs: - Introduction links to providers, hooks, getting started, platforms - API Reference links to caching, hooks, providers, configuration override - Provider pages link to caching, hooks, API Reference sections - Getting Started links to provider setup guides and secrets - GitHub Integration links to API Reference for parameters and modes - Advanced Topics pages cross-reference each other and API Reference - Fix all broken links from old directory structure - Delete old directories (examples/github-examples, advanced-topics/providers) - Run Prettier on all files Co-Authored-By: Claude Opus 4.6 --- .../01-introduction.mdx | 49 +++++++--------- .../02-game-ci-vs-orchestrator.mdx | 13 +++-- .../02-getting-started.mdx | 9 +-- .../03-examples/01-command-line.mdx | 2 - .../02-github-examples/_category_.yaml | 5 -- .../04-api-reference.mdx | 52 ++++++++--------- .../05-providers/01-overview.mdx | 36 ++++++++++++ .../02-aws.mdx | 5 +- .../03-kubernetes.mdx | 9 ++- .../05-providers/04-local-docker.mdx | 34 +++++++++++ .../05-providers/05-local.mdx | 43 ++++++++++++++ .../06-custom-providers.mdx} | 0 .../07-community-providers.mdx} | 30 +++++----- .../08-github-integration.mdx} | 12 ++-- .../09-gitlab-integration.mdx} | 6 +- .../_category_.yaml | 2 +- .../07-providers/01-overview.mdx | 57 ------------------- .../{06-advanced-topics => }/06-secrets.mdx | 2 +- .../01-caching.mdx | 9 +-- .../02-retained-workspace.mdx | 3 +- .../03-garbage-collection.mdx | 3 +- .../04-hooks}/01-custom-job.mdx | 0 .../04-hooks}/03-command-hooks.mdx | 3 + .../04-hooks}/04-container-hooks.mdx | 3 +- .../04-hooks}/05-premade-container-jobs.mdx | 0 .../04-hooks}/_category_.yaml | 2 +- .../05-configuration-override.mdx} | 0 .../06-logging.mdx} | 2 +- .../_category_.yaml | 0 29 files changed, 222 insertions(+), 169 deletions(-) delete mode 100644 docs/03-github-orchestrator/03-examples/02-github-examples/_category_.yaml create mode 100644 docs/03-github-orchestrator/05-providers/01-overview.mdx rename docs/03-github-orchestrator/{03-examples/02-github-examples => 05-providers}/02-aws.mdx (93%) rename docs/03-github-orchestrator/{03-examples/02-github-examples => 05-providers}/03-kubernetes.mdx (84%) create mode 100644 docs/03-github-orchestrator/05-providers/04-local-docker.mdx create mode 100644 docs/03-github-orchestrator/05-providers/05-local.mdx rename docs/03-github-orchestrator/{06-advanced-topics/07-providers/02-custom-providers.mdx => 05-providers/06-custom-providers.mdx} (100%) rename docs/03-github-orchestrator/{06-advanced-topics/07-providers/03-community-providers.mdx => 05-providers/07-community-providers.mdx} (75%) rename docs/03-github-orchestrator/{06-advanced-topics/07-providers/04-github-integration.mdx => 05-providers/08-github-integration.mdx} (63%) rename docs/03-github-orchestrator/{06-advanced-topics/07-providers/05-gitlab-integration.mdx => 05-providers/09-gitlab-integration.mdx} (75%) rename docs/03-github-orchestrator/{06-advanced-topics/07-providers => 05-providers}/_category_.yaml (79%) delete mode 100644 docs/03-github-orchestrator/06-advanced-topics/07-providers/01-overview.mdx rename docs/03-github-orchestrator/{06-advanced-topics => }/06-secrets.mdx (83%) rename docs/03-github-orchestrator/{06-advanced-topics => 07-advanced-topics}/01-caching.mdx (89%) rename docs/03-github-orchestrator/{06-advanced-topics => 07-advanced-topics}/02-retained-workspace.mdx (90%) rename docs/03-github-orchestrator/{06-advanced-topics => 07-advanced-topics}/03-garbage-collection.mdx (88%) rename docs/03-github-orchestrator/{06-advanced-topics/04-custom-hooks => 07-advanced-topics/04-hooks}/01-custom-job.mdx (100%) rename docs/03-github-orchestrator/{06-advanced-topics/04-custom-hooks => 07-advanced-topics/04-hooks}/03-command-hooks.mdx (81%) rename docs/03-github-orchestrator/{06-advanced-topics/04-custom-hooks => 07-advanced-topics/04-hooks}/04-container-hooks.mdx (86%) rename docs/03-github-orchestrator/{06-advanced-topics/04-custom-hooks => 07-advanced-topics/04-hooks}/05-premade-container-jobs.mdx (100%) rename docs/03-github-orchestrator/{06-advanced-topics/04-custom-hooks => 07-advanced-topics/04-hooks}/_category_.yaml (72%) rename docs/03-github-orchestrator/{06-advanced-topics/04-configuration-override.mdx => 07-advanced-topics/05-configuration-override.mdx} (100%) rename docs/03-github-orchestrator/{06-advanced-topics/05-logging.mdx => 07-advanced-topics/06-logging.mdx} (92%) rename docs/03-github-orchestrator/{06-advanced-topics => 07-advanced-topics}/_category_.yaml (100%) diff --git a/docs/03-github-orchestrator/01-introduction.mdx b/docs/03-github-orchestrator/01-introduction.mdx index 3e2af6d9..bbc87337 100644 --- a/docs/03-github-orchestrator/01-introduction.mdx +++ b/docs/03-github-orchestrator/01-introduction.mdx @@ -18,8 +18,8 @@ be built, and streams results back. │ Orchestrator handles: │ │ • Provisioning │ │ • Git sync + LFS │ - │ • Caching │ - │ • Cleanup │ + │ • Caching (S3 / rclone) │ + │ • Automatic cleanup │ └─────────────────────────────────────┘ ``` @@ -30,42 +30,35 @@ AWS Fargate and Kubernetes. 1. **Flexible and elastic** — balance speed and cost, configure CPU, memory, and disk per build 2. **Scale from zero** — no idle servers, pay only while builds run -3. **Easy setup** — minimal configuration to get started -4. **Extensible** — run custom jobs, add hooks, or bring your own provider via the plugin system +3. **Easy setup** — minimal configuration to [get started](getting-started) +4. **Extensible** — run [custom hooks](advanced-topics/hooks), or bring your own + [provider plugin](providers/custom-providers) ## ❌ When You Don't Need It - Your project is under 5 GB — standard GitHub runners should work fine - You have dedicated build servers already running -## 📦 Release Status +## 📦 Supported Providers -Orchestrator is in **active development** ⚠️ +| Cloud Provider | Description | +| -------------------------------------- | -------------------------------------------------------- | +| [AWS Fargate](providers/aws) | Fully managed containers on AWS. No servers to maintain. | +| [Kubernetes](providers/kubernetes) | Run on any Kubernetes cluster. | +| [Local Docker](providers/local-docker) | Docker containers on the local machine. | +| [Local](providers/local) | Direct execution on the host machine. | -Overall status: `preview` — core features are stable, some APIs may change. +See [Providers](providers/overview) for the full list including [custom](providers/custom-providers) +and [community](providers/community-providers) providers. -Release stages: `experimental` → `preview` → `full release` +## 🖥️ Supported Platforms -### Provider Support - -| Cloud Provider | Release Status | -| -------------- | ------------------ | -| AWS Fargate | ✔️ Full release | -| Kubernetes | ✔️ Preview release | -| GCP | ⚠️ Considered | -| Azure | ⚠️ Considered | - -See [Providers](advanced-topics/providers/overview) for details on each provider. - -### Platform Support - -| Platform | Release Status | -| ------------------ | ------------------ | -| GitHub Actions | ✔️ Full release | -| GitLab CI | ✔️ Preview release | -| Command Line | ✔️ Preview release | -| Any Git repository | ✔️ Preview release | -| Any CI system | ✔️ Preview release | +| Platform | Description | +| ---------------------------------------------- | ------------------------------------- | +| [GitHub Actions](providers/github-integration) | First-class support with Checks. | +| [GitLab CI](providers/gitlab-integration) | Via the Command Line mode. | +| [Command Line](examples/command-line) | Run from any terminal or script. | +| Any CI system | Anything that can run shell commands. | ## 🔗 External Links diff --git a/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx b/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx index 61e46eb5..59ed793a 100644 --- a/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx +++ b/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx @@ -1,3 +1,7 @@ +--- +sidebar_label: Game-CI vs Orchestrator +--- + # Standard Game-CI vs Orchestrator Mode ## 🎮 Standard Game-CI @@ -20,10 +24,11 @@ the CI runner itself. This is useful when: Standard Game-CI Orchestrator Mode ┌──────────────┐ ┌──────────────┐ ┌────────────┐ - │ GitHub │ │ GitHub │ ───► │ Cloud │ - │ Runner │ │ Runner │ │ Container │ - │ (builds │ │ (dispatches │ ◄─── │ (builds │ - │ locally) │ │ only) │ │ remotely) │ + │ GitHub │ │ GitHub Action │ │ │ + │ Runner │ │ CLI / Any CI │ ───► │ Cloud │ + │ (builds │ │ │ │ Container │ + │ locally) │ │ (dispatches │ ◄─── │ (builds │ + │ │ │ only) │ │ remotely) │ └──────────────┘ └──────────────┘ └────────────┘ ~14 GB disk Configurable CPU, memory, disk Fixed resources Scales to zero when idle diff --git a/docs/03-github-orchestrator/02-getting-started.mdx b/docs/03-github-orchestrator/02-getting-started.mdx index f14c1a59..def9b8ce 100644 --- a/docs/03-github-orchestrator/02-getting-started.mdx +++ b/docs/03-github-orchestrator/02-getting-started.mdx @@ -7,7 +7,7 @@ runners. This is useful for large projects that exceed GitHub's disk or resource - A Unity project in a GitHub repository - A cloud provider account (AWS or a Kubernetes cluster) -- Provider credentials configured as GitHub secrets +- Provider credentials configured as GitHub [secrets](secrets) ## Quick Start @@ -15,10 +15,11 @@ runners. This is useful for large projects that exceed GitHub's disk or resource 2. **Configure credentials** for your chosen provider 3. **Add the orchestrator step** to your workflow -See the provider-specific examples for complete setup: +See the provider-specific setup guides: -- [AWS Example](examples/github-examples/aws) -- [Kubernetes Example](examples/github-examples/kubernetes) +- [AWS Fargate](providers/aws) +- [Kubernetes](providers/kubernetes) +- [Local Docker](providers/local-docker) - [Command Line](examples/command-line) ## Minimal Example diff --git a/docs/03-github-orchestrator/03-examples/01-command-line.mdx b/docs/03-github-orchestrator/03-examples/01-command-line.mdx index 17b7f4ec..72e81bac 100644 --- a/docs/03-github-orchestrator/03-examples/01-command-line.mdx +++ b/docs/03-github-orchestrator/03-examples/01-command-line.mdx @@ -1,7 +1,5 @@ # Command Line -_Preview Support Only_ - You can install Game CI locally and start orchestrator jobs from the command line or by integrating your own tools. All parameters in the [API Reference](../api-reference) can be specified as CLI flags. diff --git a/docs/03-github-orchestrator/03-examples/02-github-examples/_category_.yaml b/docs/03-github-orchestrator/03-examples/02-github-examples/_category_.yaml deleted file mode 100644 index b9807118..00000000 --- a/docs/03-github-orchestrator/03-examples/02-github-examples/_category_.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -position: 2.0 -label: GitHub -collapsible: true -collapsed: true diff --git a/docs/03-github-orchestrator/04-api-reference.mdx b/docs/03-github-orchestrator/04-api-reference.mdx index 45b6a31e..87645484 100644 --- a/docs/03-github-orchestrator/04-api-reference.mdx +++ b/docs/03-github-orchestrator/04-api-reference.mdx @@ -31,13 +31,13 @@ Set the mode to control what Orchestrator does. Default: `cli-build`. ### Provider -| Parameter | Default | Description | -| ---------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `providerStrategy` | `local` | Cloud provider to use. Built-in: `aws`, `k8s`, `local-docker`, `local`. Also accepts a GitHub URL, NPM package, or local path for [custom providers](advanced-topics/providers/custom-providers). | -| `containerCpu` | `1024` | CPU units for cloud containers (`1024` = 1 vCPU). See provider setup guides for allowed values. | -| `containerMemory` | `3072` | Memory in MB for cloud containers (`4096` = 4 GB). See provider setup guides for allowed values. | -| `orchestratorBranch` | `main` | Release branch of Orchestrator for remote containers. Use `orchestrator-develop` for latest development builds. | -| `orchestratorRepoName` | `game-ci/unity-builder` | Repository for Orchestrator source. Override to use a fork for testing or custom builds. | +| Parameter | Default | Description | +| ---------------------- | ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `providerStrategy` | `local` | Cloud provider to use. Built-in: `aws`, `k8s`, `local-docker`, `local`. Also accepts a GitHub URL, NPM package, or local path for [custom providers](providers/custom-providers). | +| `containerCpu` | `1024` | CPU units for cloud containers (`1024` = 1 vCPU). See provider setup guides for allowed values. | +| `containerMemory` | `3072` | Memory in MB for cloud containers (`4096` = 4 GB). See provider setup guides for allowed values. | +| `orchestratorBranch` | `main` | Release branch of Orchestrator for remote containers. Use `orchestrator-develop` for latest development builds. | +| `orchestratorRepoName` | `game-ci/unity-builder` | Repository for Orchestrator source. Override to use a fork for testing or custom builds. | ### Git Synchronization @@ -52,16 +52,16 @@ Set the mode to control what Orchestrator does. Default: `cli-build`. ### Custom Hooks -| Parameter | Default | Description | -| ------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------ | -| `containerHookFiles` | — | Names of [container hook](advanced-topics/custom-hooks/container-hooks) files from `.game-ci/container-hooks/`. | -| `customHookFiles` | — | Names of custom hook files from `.game-ci/hooks/`. | -| `customCommandHooks` | — | Inline [command hooks](advanced-topics/custom-hooks/command-hooks) as YAML. | -| `postBuildSteps` | — | Post-build job in YAML format with keys: `image`, `secrets`, `command`. | -| `preBuildSteps` | — | Pre-build job (after repo setup, before build) in YAML format. | -| `postBuildContainerHooks` | — | Container hook files to run after the build step. | -| `preBuildContainerHooks` | — | Container hook files to run before the build step. | -| `customJob` | — | Custom job YAML to override the default build workflow. See [Custom Job](advanced-topics/custom-hooks/custom-job). | +| Parameter | Default | Description | +| ------------------------- | ------- | ----------------------------------------------------------------------------------------------------------- | +| `containerHookFiles` | — | Names of [container hook](advanced-topics/hooks/container-hooks) files from `.game-ci/container-hooks/`. | +| `customHookFiles` | — | Names of custom hook files from `.game-ci/hooks/`. | +| `customCommandHooks` | — | Inline [command hooks](advanced-topics/hooks/command-hooks) as YAML. | +| `postBuildSteps` | — | Post-build job in YAML format with keys: `image`, `secrets`, `command`. | +| `preBuildSteps` | — | Pre-build job (after repo setup, before build) in YAML format. | +| `postBuildContainerHooks` | — | Container hook files to run after the build step. | +| `preBuildContainerHooks` | — | Container hook files to run before the build step. | +| `customJob` | — | Custom job YAML to override the default build workflow. See [Custom Job](advanced-topics/hooks/custom-job). | ### Configuration Override @@ -72,10 +72,10 @@ Set the mode to control what Orchestrator does. Default: `cli-build`. ### Storage -| Parameter | Default | Description | -| ----------------- | ------- | --------------------------------------------------------------------------- | -| `storageProvider` | `s3` | Storage backend for caching and artifacts. Accepted values: `s3`, `rclone`. | -| `rcloneRemote` | — | Rclone remote endpoint. Required when `storageProvider` is `rclone`. | +| Parameter | Default | Description | +| ----------------- | ------- | ------------------------------------------------------------------------------------------------------ | +| `storageProvider` | `s3` | Storage backend for [caching](advanced-topics/caching) and artifacts. Accepted values: `s3`, `rclone`. | +| `rcloneRemote` | — | Rclone remote endpoint. Required when `storageProvider` is `rclone`. | ### AWS @@ -107,11 +107,11 @@ Set the mode to control what Orchestrator does. Default: `cli-build`. ### GitHub Integration -| Parameter | Default | Description | -| ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------- | -| `githubCheck` | `false` | Create a GitHub Check for each orchestrator step. See [GitHub Integration](advanced-topics/providers/github-integration). | -| `asyncOrchestrator` | `false` | Run in async mode — returns immediately without waiting for the build to complete. | -| `watchToEnd` | `true` | Whether to follow the build logs until completion. | +| Parameter | Default | Description | +| ------------------- | ------- | --------------------------------------------------------------------------------------------------------- | +| `githubCheck` | `false` | Create a GitHub Check for each orchestrator step. See [GitHub Integration](providers/github-integration). | +| `asyncOrchestrator` | `false` | Run in async mode — returns immediately without waiting for the build to complete. | +| `watchToEnd` | `true` | Whether to follow the build logs until completion. | ### Build Options diff --git a/docs/03-github-orchestrator/05-providers/01-overview.mdx b/docs/03-github-orchestrator/05-providers/01-overview.mdx new file mode 100644 index 00000000..a5094b96 --- /dev/null +++ b/docs/03-github-orchestrator/05-providers/01-overview.mdx @@ -0,0 +1,36 @@ +# Providers + +A **provider** is the backend that Orchestrator uses to run your builds. You choose a provider by +setting the `providerStrategy` parameter. + +## Built-in Providers + +These providers ship with Orchestrator and are maintained by the Game CI team. + +| Provider | `providerStrategy` | Description | +| -------------- | ------------------ | ----------------------------------------------------------------------------- | +| AWS Fargate | `aws` | Runs jobs on AWS Fargate (ECS). Fully managed, no servers to maintain. | +| Kubernetes | `k8s` | Runs jobs on any Kubernetes cluster. Flexible but requires a running cluster. | +| Local Docker | `local-docker` | Runs jobs in Docker containers on the local machine. | +| Local (direct) | `local` | Runs jobs directly on the local machine without containers. | + +Each provider has its own page with setup instructions: + +- [AWS Fargate](aws) +- [Kubernetes](kubernetes) +- [Local Docker](local-docker) +- [Local](local) + +## Custom Providers + +Extend Orchestrator with your own provider by pointing `providerStrategy` at a GitHub repository, +NPM package, or local file path. + +See [Custom Providers](custom-providers) for the full guide. + +## Community Providers + +Third-party providers shared by the Game CI community. + +See the [Community Providers](community-providers) page for the current list and how to submit your +own. diff --git a/docs/03-github-orchestrator/03-examples/02-github-examples/02-aws.mdx b/docs/03-github-orchestrator/05-providers/02-aws.mdx similarity index 93% rename from docs/03-github-orchestrator/03-examples/02-github-examples/02-aws.mdx rename to docs/03-github-orchestrator/05-providers/02-aws.mdx index 442dad1e..21ce26dc 100644 --- a/docs/03-github-orchestrator/03-examples/02-github-examples/02-aws.mdx +++ b/docs/03-github-orchestrator/05-providers/02-aws.mdx @@ -62,8 +62,7 @@ Common combinations: containerHookFiles: aws-s3-upload-build ``` -See [Container Hooks](../../advanced-topics/custom-hooks/container-hooks) for more on -`containerHookFiles`. +See [Container Hooks](../advanced-topics/hooks/container-hooks) for more on `containerHookFiles`. A full workflow example is available in the builder source: [orchestrator-pipeline.yml](https://github.com/game-ci/unity-builder/blob/main/.github/workflows/orchestrator-pipeline.yml). @@ -71,4 +70,4 @@ A full workflow example is available in the builder source: ## AWS Parameters For the full list of AWS-specific parameters (`awsStackName`, endpoint overrides, etc.), see the -[API Reference — AWS section](../../api-reference#aws). +[API Reference — AWS section](../api-reference#aws). diff --git a/docs/03-github-orchestrator/03-examples/02-github-examples/03-kubernetes.mdx b/docs/03-github-orchestrator/05-providers/03-kubernetes.mdx similarity index 84% rename from docs/03-github-orchestrator/03-examples/02-github-examples/03-kubernetes.mdx rename to docs/03-github-orchestrator/05-providers/03-kubernetes.mdx index f5b5faf1..5453a8e2 100644 --- a/docs/03-github-orchestrator/03-examples/02-github-examples/03-kubernetes.mdx +++ b/docs/03-github-orchestrator/05-providers/03-kubernetes.mdx @@ -46,8 +46,7 @@ vCPU or GB suffix. containerHookFiles: aws-s3-upload-build ``` -See [Container Hooks](../../advanced-topics/custom-hooks/container-hooks) for more on -`containerHookFiles`. +See [Container Hooks](../advanced-topics/hooks/container-hooks) for more on `containerHookFiles`. A full workflow example is available in the builder source: [orchestrator-k8s-pipeline.yml](https://github.com/game-ci/unity-builder/blob/main/.github/workflows/orchestrator-k8s-pipeline.yml). @@ -56,10 +55,10 @@ A full workflow example is available in the builder source: - **Keep the cluster running.** Cold-starting a Kubernetes cluster is slow. If you need auto-scaling to zero, consider Google Cloud Kubernetes Autopilot. -- **Cloud storage required.** Kubernetes support currently requires cloud storage for caching. S3 is - built-in, or use rclone for other backends. +- **Cloud storage required.** Kubernetes requires cloud storage for + [caching](../advanced-topics/caching). S3 is built-in, or use rclone for other backends. ## K8s Parameters For the full list of Kubernetes parameters (`kubeConfig`, `kubeVolume`, `kubeStorageClass`, etc.), -see the [API Reference — Kubernetes section](../../api-reference#kubernetes). +see the [API Reference — Kubernetes section](../api-reference#kubernetes). diff --git a/docs/03-github-orchestrator/05-providers/04-local-docker.mdx b/docs/03-github-orchestrator/05-providers/04-local-docker.mdx new file mode 100644 index 00000000..0d20acf9 --- /dev/null +++ b/docs/03-github-orchestrator/05-providers/04-local-docker.mdx @@ -0,0 +1,34 @@ +# Local Docker + +Runs the build workflow inside a Docker container on the local machine. No cloud account required. + +## Requirements + +- Docker installed and running on the build machine. + +## Example Workflow + +### GitHub Actions (self-hosted runner) + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: local-docker + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +### Command Line + +```bash +yarn run cli -m cli-build \ + --providerStrategy local-docker \ + --projectPath /path/to/your/project \ + --targetPlatform StandaloneLinux64 +``` + +## When to Use + +- You have a self-hosted runner with Docker installed +- You want container isolation without cloud infrastructure +- Testing builds locally before deploying to AWS or Kubernetes diff --git a/docs/03-github-orchestrator/05-providers/05-local.mdx b/docs/03-github-orchestrator/05-providers/05-local.mdx new file mode 100644 index 00000000..14287da2 --- /dev/null +++ b/docs/03-github-orchestrator/05-providers/05-local.mdx @@ -0,0 +1,43 @@ +# Local + +Runs builds directly on the host machine with no container isolation. The simplest provider — useful +for development and testing. + +## Requirements + +- Unity installed on the build machine. +- No Docker or cloud account needed. + +## Example Workflow + +### GitHub Actions (self-hosted runner) + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: local + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +### Command Line + +```bash +yarn run cli -m cli-build \ + --providerStrategy local \ + --projectPath /path/to/your/project \ + --targetPlatform StandaloneLinux64 +``` + +## When to Use + +- Local development and testing of Orchestrator workflows +- Debugging build issues before deploying to cloud providers +- Self-hosted runners where you want direct execution without containers + +## ⚠️ Notes + +- Builds run directly on the host with no isolation. Ensure the machine has the required Unity + version and dependencies installed. +- This is the fallback provider — if a custom provider fails to load, Orchestrator falls back to + `local`. diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/02-custom-providers.mdx b/docs/03-github-orchestrator/05-providers/06-custom-providers.mdx similarity index 100% rename from docs/03-github-orchestrator/06-advanced-topics/07-providers/02-custom-providers.mdx rename to docs/03-github-orchestrator/05-providers/06-custom-providers.mdx diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/03-community-providers.mdx b/docs/03-github-orchestrator/05-providers/07-community-providers.mdx similarity index 75% rename from docs/03-github-orchestrator/06-advanced-topics/07-providers/03-community-providers.mdx rename to docs/03-github-orchestrator/05-providers/07-community-providers.mdx index 213aaed9..7e5b7ed4 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/07-providers/03-community-providers.mdx +++ b/docs/03-github-orchestrator/05-providers/07-community-providers.mdx @@ -15,27 +15,27 @@ provider before using it in your pipelines. _No community providers have been submitted yet. Yours could be the first!_ -{/\* When adding a provider, use this format: - -### Provider Name - -| | | -| -------------------- | -------------------------------------------------------------------------- | -| **Repository** | [user/repo](https://github.com/user/repo) | -| **providerStrategy** | `user/repo` | -| **Description** | Brief description of what the provider does and which platform it targets. | -| **Maintainer** | [@username](https://github.com/username) | - -\*/} - ## Submit Your Provider Built a custom provider? Share it with the community by adding it to this page. -1. [Edit this file directly on GitHub](https://github.com/game-ci/documentation/edit/main/docs/03-github-orchestrator/06-advanced-topics/07-providers/03-community-providers.mdx) -2. Add your provider to the **Provider List** section above using the commented template +1. [Edit this file directly on GitHub](https://github.com/game-ci/documentation/edit/main/docs/03-github-orchestrator/05-providers/07-community-providers.mdx) +2. Add your provider using the template below 3. Submit a pull request for review +### Template + +```markdown +### Provider Name + +| | | +| -------------------- | -------------------------------------------- | +| **Repository** | [user/repo](https://github.com/user/repo) | +| **providerStrategy** | `user/repo` | +| **Description** | Brief description of what the provider does. | +| **Maintainer** | [@username](https://github.com/username) | +``` + Your provider should: - Have a public GitHub repository or published NPM package diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/04-github-integration.mdx b/docs/03-github-orchestrator/05-providers/08-github-integration.mdx similarity index 63% rename from docs/03-github-orchestrator/06-advanced-topics/07-providers/04-github-integration.mdx rename to docs/03-github-orchestrator/05-providers/08-github-integration.mdx index 989573a6..ca43528c 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/07-providers/04-github-integration.mdx +++ b/docs/03-github-orchestrator/05-providers/08-github-integration.mdx @@ -6,8 +6,8 @@ environment. ## GitHub Checks -By enabling the `githubCheck` parameter, the orchestrator job will create a GitHub check for each -step. +By enabling the [`githubCheck`](../api-reference#github-integration) parameter, the orchestrator job +will create a GitHub check for each step. ```yaml - uses: game-ci/unity-builder@v4 @@ -24,8 +24,9 @@ UI. ## Async Mode -Set `asyncOrchestrator: true` to start a build without waiting for it to complete. The GitHub Action -will return immediately and you can check progress via GitHub Checks or by running the `watch` mode. +Set [`asyncOrchestrator: true`](../api-reference#github-integration) to start a build without +waiting for it to complete. The GitHub Action will return immediately and you can check progress via +GitHub Checks or by running the [`watch` mode](../api-reference#modes). ```yaml - uses: game-ci/unity-builder@v4 @@ -37,5 +38,4 @@ will return immediately and you can check progress via GitHub Checks or by runni gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} ``` -See the [AWS](../../examples/github-examples/aws) and -[Kubernetes](../../examples/github-examples/kubernetes) example pages for full workflow files. +See the [AWS](aws) and [Kubernetes](kubernetes) provider pages for full workflow files. diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/05-gitlab-integration.mdx b/docs/03-github-orchestrator/05-providers/09-gitlab-integration.mdx similarity index 75% rename from docs/03-github-orchestrator/06-advanced-topics/07-providers/05-gitlab-integration.mdx rename to docs/03-github-orchestrator/05-providers/09-gitlab-integration.mdx index f7b59864..d1113e51 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/07-providers/05-gitlab-integration.mdx +++ b/docs/03-github-orchestrator/05-providers/09-gitlab-integration.mdx @@ -5,7 +5,7 @@ GitHub Actions — any CI system that can run shell commands can trigger orchest ## Setup -1. Install the Orchestrator CLI (see [Command Line](../../examples/command-line)) +1. Install the Orchestrator CLI (see [Command Line](../examples/command-line)) 2. Set your git credentials and provider configuration as environment variables 3. Run the CLI from your `.gitlab-ci.yml` pipeline @@ -26,5 +26,5 @@ build-unity: AWS_DEFAULT_REGION: eu-west-2 ``` -See the [Command Line](../../examples/command-line) page for more details on CLI usage and the -[Configuration Override](../configuration-override) page for managing credentials. +See the [Command Line](../examples/command-line) page for more details on CLI usage and the +[Configuration Override](../advanced-topics/configuration-override) page for managing credentials. diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/_category_.yaml b/docs/03-github-orchestrator/05-providers/_category_.yaml similarity index 79% rename from docs/03-github-orchestrator/06-advanced-topics/07-providers/_category_.yaml rename to docs/03-github-orchestrator/05-providers/_category_.yaml index 280c7133..94aafd4d 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/07-providers/_category_.yaml +++ b/docs/03-github-orchestrator/05-providers/_category_.yaml @@ -1,5 +1,5 @@ --- -position: 7.0 +position: 5.0 label: Providers collapsible: true collapsed: true diff --git a/docs/03-github-orchestrator/06-advanced-topics/07-providers/01-overview.mdx b/docs/03-github-orchestrator/06-advanced-topics/07-providers/01-overview.mdx deleted file mode 100644 index 217c17bf..00000000 --- a/docs/03-github-orchestrator/06-advanced-topics/07-providers/01-overview.mdx +++ /dev/null @@ -1,57 +0,0 @@ -# Providers - -A **provider** is the backend that Orchestrator uses to run your builds. You choose a provider by -setting the `providerStrategy` parameter. - -## Built-in Providers - -These providers ship with Orchestrator and are maintained by the Game CI team. - -| Provider | `providerStrategy` | Description | Release Status | -| -------------- | ------------------ | ----------------------------------------------------------------------------- | -------------- | -| AWS Fargate | `aws` | Runs jobs on AWS Fargate (ECS). Fully managed, no servers to maintain. | Full release | -| Kubernetes | `k8s` | Runs jobs on any Kubernetes cluster. Flexible but requires a running cluster. | Preview | -| Local Docker | `local-docker` | Runs jobs in Docker containers on the local machine. | Preview | -| Local (direct) | `local` | Runs jobs directly on the local machine without containers. | Preview | - -### AWS Fargate - -Best for most users. Orchestrator automatically provisions and tears down Fargate tasks — you only -pay while a build is running. Requires an AWS account with IAM credentials. - -See [AWS setup guide](../../examples/github-examples/aws) for credentials and configuration. - -### Kubernetes - -Run builds on your own Kubernetes cluster. The cluster must support persistent volumes and should -stay running (cold starts are slow). Google Cloud Kubernetes Autopilot can scale to near-zero when -idle. - -See [Kubernetes setup guide](../../examples/github-examples/kubernetes) for credentials and -configuration. - -### Local Docker - -Runs the build workflow inside a Docker container on the machine running the action. Useful for -self-hosted runners with Docker installed. - -### Local - -Runs builds directly on the host machine with no container isolation. Useful for testing and -development. - -## Custom Providers - -You can extend Orchestrator with your own provider by pointing `providerStrategy` at a GitHub -repository, NPM package, or local file path. Orchestrator will dynamically load and validate it at -runtime. - -See [Custom Providers](custom-providers) for the full guide on creating and using custom providers. - -## Community Providers - -Community providers are third-party providers shared by the Game CI community. They are not -maintained by the Game CI team but have been reviewed and listed here for discoverability. - -See the [Community Providers](community-providers) page for the current list and instructions on how -to submit your own. diff --git a/docs/03-github-orchestrator/06-advanced-topics/06-secrets.mdx b/docs/03-github-orchestrator/06-secrets.mdx similarity index 83% rename from docs/03-github-orchestrator/06-advanced-topics/06-secrets.mdx rename to docs/03-github-orchestrator/06-secrets.mdx index 2c04fbb5..56631718 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/06-secrets.mdx +++ b/docs/03-github-orchestrator/06-secrets.mdx @@ -16,4 +16,4 @@ variables. ## 🔐 Managing Secrets For pulling secrets from external sources (e.g. GCP Secret Manager, AWS Secrets Manager) into -Orchestrator parameters, see [Configuration Override](configuration-override). +Orchestrator parameters, see [Configuration Override](advanced-topics/configuration-override). diff --git a/docs/03-github-orchestrator/06-advanced-topics/01-caching.mdx b/docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx similarity index 89% rename from docs/03-github-orchestrator/06-advanced-topics/01-caching.mdx rename to docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx index a7eb3563..40ee1ef6 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/01-caching.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx @@ -1,7 +1,8 @@ # Caching -Orchestrator supports two caching strategies. You can mix both by setting `maxRetainedWorkspaces` — -once the limit is reached, additional jobs fall back to standard caching. +Orchestrator supports two caching strategies. You can mix both by setting +[`maxRetainedWorkspaces`](../api-reference#caching) — once the limit is reached, additional jobs +fall back to standard caching. ``` Standard Caching Retained Workspace @@ -42,8 +43,8 @@ See [Retained Workspaces](retained-workspace) for configuration details. | S3 | `s3` (default) | AWS S3 storage. Works with both AWS and LocalStack. | | Rclone | `rclone` | Flexible cloud storage via [rclone](https://rclone.org). Supports 70+ backends (Google Cloud, Azure Blob, Backblaze, SFTP, etc). | -Configure with the `storageProvider` parameter. When using rclone, also set `rcloneRemote` to your -configured remote endpoint. +Configure with the [`storageProvider`](../api-reference#storage) parameter. When using rclone, also +set `rcloneRemote` to your configured remote endpoint. ## 🔒 Workspace Locking diff --git a/docs/03-github-orchestrator/06-advanced-topics/02-retained-workspace.mdx b/docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx similarity index 90% rename from docs/03-github-orchestrator/06-advanced-topics/02-retained-workspace.mdx rename to docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx index cce787cf..ab998997 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/02-retained-workspace.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx @@ -13,7 +13,8 @@ Set `maxRetainedWorkspaces` to control how many workspaces are kept: | `> 0` | Keep at most N workspaces. Additional jobs fall back to standard caching. | Each retained workspace is locked during use — only one build can use a workspace at a time. -Orchestrator handles locking automatically via S3 or rclone. +Orchestrator handles locking automatically via S3 or rclone. See [Caching](caching) for storage +provider details. ## Example diff --git a/docs/03-github-orchestrator/06-advanced-topics/03-garbage-collection.mdx b/docs/03-github-orchestrator/07-advanced-topics/03-garbage-collection.mdx similarity index 88% rename from docs/03-github-orchestrator/06-advanced-topics/03-garbage-collection.mdx rename to docs/03-github-orchestrator/07-advanced-topics/03-garbage-collection.mdx index 6400f1f1..140c036a 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/03-garbage-collection.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/03-garbage-collection.mdx @@ -3,7 +3,8 @@ Orchestrator creates cloud resources (containers, stacks, volumes) for each build and cleans them up automatically. If a build fails or is interrupted, resources may be left behind. -Use garbage collection to clean up stale resources. +Use garbage collection to clean up stale resources. See the +[API Reference](../api-reference#garbage-collection) for all parameters. ## Usage diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/01-custom-job.mdx b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/01-custom-job.mdx similarity index 100% rename from docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/01-custom-job.mdx rename to docs/03-github-orchestrator/07-advanced-topics/04-hooks/01-custom-job.mdx diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/03-command-hooks.mdx b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/03-command-hooks.mdx similarity index 81% rename from docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/03-command-hooks.mdx rename to docs/03-github-orchestrator/07-advanced-topics/04-hooks/03-command-hooks.mdx index 8a4327e7..918420f6 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/03-command-hooks.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/03-command-hooks.mdx @@ -35,3 +35,6 @@ directory via `customHookFiles`. ``` Place hook files at `.game-ci/hooks/my-hook.yaml` in your repository. + +For running Docker containers as build steps, see [Container Hooks](container-hooks). For +ready-to-use hooks (S3, rclone, Steam), see [Premade Container Hooks](premade-container-jobs). diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/04-container-hooks.mdx b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx similarity index 86% rename from docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/04-container-hooks.mdx rename to docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx index 70bcdeea..0931539c 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/04-container-hooks.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx @@ -1,7 +1,8 @@ # Container Hooks Run custom Docker containers as steps in the build workflow. Useful for uploading artifacts, -deploying builds, or running additional tools. +deploying builds, or running additional tools. For inline shell commands instead, see +[Command Hooks](command-hooks). ## Format diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/05-premade-container-jobs.mdx b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/05-premade-container-jobs.mdx similarity index 100% rename from docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/05-premade-container-jobs.mdx rename to docs/03-github-orchestrator/07-advanced-topics/04-hooks/05-premade-container-jobs.mdx diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/_category_.yaml b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/_category_.yaml similarity index 72% rename from docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/_category_.yaml rename to docs/03-github-orchestrator/07-advanced-topics/04-hooks/_category_.yaml index 445f7ab4..6983ed19 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/04-custom-hooks/_category_.yaml +++ b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/_category_.yaml @@ -1,5 +1,5 @@ --- position: 4.0 -label: Custom Hooks +label: Hooks collapsible: true collapsed: true diff --git a/docs/03-github-orchestrator/06-advanced-topics/04-configuration-override.mdx b/docs/03-github-orchestrator/07-advanced-topics/05-configuration-override.mdx similarity index 100% rename from docs/03-github-orchestrator/06-advanced-topics/04-configuration-override.mdx rename to docs/03-github-orchestrator/07-advanced-topics/05-configuration-override.mdx diff --git a/docs/03-github-orchestrator/06-advanced-topics/05-logging.mdx b/docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx similarity index 92% rename from docs/03-github-orchestrator/06-advanced-topics/05-logging.mdx rename to docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx index c044820d..843d578b 100644 --- a/docs/03-github-orchestrator/06-advanced-topics/05-logging.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx @@ -26,7 +26,7 @@ Logs flow through a CloudWatch → Kinesis pipeline: ## 🐛 Debug Logging -Enable `orchestratorDebug: true` to get verbose output including: +Enable [`orchestratorDebug: true`](../api-reference#build-options) to get verbose output including: - Resource allocation summaries (CPU, memory, disk) - Directory structure via `tree` diff --git a/docs/03-github-orchestrator/06-advanced-topics/_category_.yaml b/docs/03-github-orchestrator/07-advanced-topics/_category_.yaml similarity index 100% rename from docs/03-github-orchestrator/06-advanced-topics/_category_.yaml rename to docs/03-github-orchestrator/07-advanced-topics/_category_.yaml From 07229a6a6a740dd9e72fb2b9247e943f08e380ad Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:10:12 +0000 Subject: [PATCH 05/20] Merge Configuration Override into Secrets page, rename to Pull Secrets - Merge configuration-override.mdx content into secrets.mdx as a section - Delete standalone configuration-override page - Rename "Configuration Override" to "Pull Secrets" in API reference - Update all cross-links (command-line, GitLab integration, API reference) - Fix logging: "Orchestrator job (Fargate task)" instead of "Fargate tasks" Co-Authored-By: Claude Opus 4.6 --- .../03-examples/01-command-line.mdx | 4 +- .../04-api-reference.mdx | 36 +++++++-------- .../05-providers/09-gitlab-integration.mdx | 2 +- docs/03-github-orchestrator/06-secrets.mdx | 44 +++++++++++++++++-- .../05-configuration-override.mdx | 42 ------------------ .../07-advanced-topics/06-logging.mdx | 2 +- 6 files changed, 63 insertions(+), 67 deletions(-) delete mode 100644 docs/03-github-orchestrator/07-advanced-topics/05-configuration-override.mdx diff --git a/docs/03-github-orchestrator/03-examples/01-command-line.mdx b/docs/03-github-orchestrator/03-examples/01-command-line.mdx index 72e81bac..9159f7d3 100644 --- a/docs/03-github-orchestrator/03-examples/01-command-line.mdx +++ b/docs/03-github-orchestrator/03-examples/01-command-line.mdx @@ -55,9 +55,9 @@ Run `yarn run cli --help` to list all modes and parameters with descriptions. ## Keeping Commands Short You can avoid long CLI flags for credentials by using environment variables or the -[Configuration Override](../advanced-topics/configuration-override#example) feature. +[Pull Secrets](../secrets#-pulling-secrets-from-external-sources) feature. -This lets you pull input from a file or secret manager: +This lets you pull input from a secret manager: ```bash yarn run cli \ diff --git a/docs/03-github-orchestrator/04-api-reference.mdx b/docs/03-github-orchestrator/04-api-reference.mdx index 87645484..3e72974a 100644 --- a/docs/03-github-orchestrator/04-api-reference.mdx +++ b/docs/03-github-orchestrator/04-api-reference.mdx @@ -2,12 +2,12 @@ ## ⚙️ Configuration Methods -| Method | Description | -| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| **GitHub Action `with`** | Pass parameters directly in your workflow file. See [Getting Started](getting-started). | -| **Command Line** | Pass parameters as CLI flags. See [Command Line](examples/command-line). | -| **Environment Variables** | Set parameters as environment variables in your shell or CI environment. | -| **Configuration Override** | Pull parameters dynamically from secret managers or files. See [Configuration Override](advanced-topics/configuration-override). | +| Method | Description | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| **GitHub Action `with`** | Pass parameters directly in your workflow file. See [Getting Started](getting-started). | +| **Command Line** | Pass parameters as CLI flags. See [Command Line](examples/command-line). | +| **Environment Variables** | Set parameters as environment variables in your shell or CI environment. | +| **Pull Secrets** | Pull parameters dynamically from secret managers or files. See [Secrets](secrets#-pulling-secrets-from-external-sources). | ## 🔧 Modes @@ -63,12 +63,12 @@ Set the mode to control what Orchestrator does. Default: `cli-build`. | `preBuildContainerHooks` | — | Container hook files to run before the build step. | | `customJob` | — | Custom job YAML to override the default build workflow. See [Custom Job](advanced-topics/hooks/custom-job). | -### Configuration Override +### Pull Secrets -| Parameter | Default | Description | -| --------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `readInputOverrideCommand` | — | Command to read a parameter value from an external source. Use `{0}` as the parameter name placeholder. Built-in presets: `gcp-secret-manager`, `aws-secret-manager`. See [Configuration Override](advanced-topics/configuration-override). | -| `readInputFromOverrideList` | — | Comma-separated list of parameter names to pull via `readInputOverrideCommand`. | +| Parameter | Default | Description | +| --------------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `readInputOverrideCommand` | — | Command to read a parameter value from an external source. Use `{0}` as the parameter name placeholder. Built-in presets: `gcp-secret-manager`, `aws-secret-manager`. See [Secrets](secrets). | +| `readInputFromOverrideList` | — | Comma-separated list of parameter names to pull via `readInputOverrideCommand`. | ### Storage @@ -134,13 +134,13 @@ Set the mode to control what Orchestrator does. Default: `cli-build`. These parameters are only available when using Orchestrator from the command line. -| Parameter | Description | -| ------------------ | ---------------------------------------------------- | -| `populateOverride` | Enable reading parameters from the override command. | -| `cachePushFrom` | Local directory to push cache from. | -| `cachePushTo` | Remote path to push cache to. | -| `artifactName` | Name for the build artifact. | -| `select` | Select a specific workflow or resource by name. | +| Parameter | Description | +| ------------------ | -------------------------------------------------------------------------------------------------- | +| `populateOverride` | Enable [pulling secrets](secrets#-pulling-secrets-from-external-sources) from an external command. | +| `cachePushFrom` | Local directory to push cache from. | +| `cachePushTo` | Remote path to push cache to. | +| `artifactName` | Name for the build artifact. | +| `select` | Select a specific workflow or resource by name. | ## 🌍 Environment Variables diff --git a/docs/03-github-orchestrator/05-providers/09-gitlab-integration.mdx b/docs/03-github-orchestrator/05-providers/09-gitlab-integration.mdx index d1113e51..a6d5d944 100644 --- a/docs/03-github-orchestrator/05-providers/09-gitlab-integration.mdx +++ b/docs/03-github-orchestrator/05-providers/09-gitlab-integration.mdx @@ -27,4 +27,4 @@ build-unity: ``` See the [Command Line](../examples/command-line) page for more details on CLI usage and the -[Configuration Override](../advanced-topics/configuration-override) page for managing credentials. +[Secrets](../secrets) page for managing credentials. diff --git a/docs/03-github-orchestrator/06-secrets.mdx b/docs/03-github-orchestrator/06-secrets.mdx index 56631718..d79182cb 100644 --- a/docs/03-github-orchestrator/06-secrets.mdx +++ b/docs/03-github-orchestrator/06-secrets.mdx @@ -13,7 +13,45 @@ job containers. Secrets are stored in **AWS Secrets Manager** and injected into Fargate tasks as environment variables. -## 🔐 Managing Secrets +## 🔐 Pulling Secrets from External Sources -For pulling secrets from external sources (e.g. GCP Secret Manager, AWS Secrets Manager) into -Orchestrator parameters, see [Configuration Override](advanced-topics/configuration-override). +You can pull parameter values from external secret managers or files at runtime instead of +hardcoding credentials. This keeps CLI commands short and secrets out of your repository. + +``` + Orchestrator External Source + ┌──────────────┐ command ┌──────────────────┐ + │ Reads input │ ──────────────► │ Secret Manager │ + │ override │ │ (GCP, AWS, file) │ + │ list │ ◄────────────── │ │ + │ │ value │ │ + └──────────────┘ └──────────────────┘ +``` + +### Parameters + +| Parameter | Default | Description | +| --------------------------- | ------- | ----------------------------------------------------------------------------------------------------- | +| `readInputOverrideCommand` | — | Command to run for each secret. Use `{0}` as a placeholder for the parameter name. | +| `readInputFromOverrideList` | — | Comma-separated list of parameter names to pull via `readInputOverrideCommand`. | +| `populateOverride` | — | Must be `true` to enable pulling secrets (CLI only). Auto-enabled in GitHub Actions when command set. | + +### Built-in Presets + +Instead of writing a full command, use these presets as the `readInputOverrideCommand`: + +| Preset | Expands to | +| -------------------- | ----------------------------------------------------- | +| `gcp-secret-manager` | `gcloud secrets versions access 1 --secret="{0}"` | +| `aws-secret-manager` | `aws secretsmanager get-secret-value --secret-id {0}` | + +### Example + +```bash +yarn run cli -m cli-build \ + --populateOverride true \ + --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD \ + --readInputOverrideCommand='gcloud secrets versions access 1 --secret="{0}"' +``` + +This runs the GCP command for each parameter name in the list and uses the output as the value. diff --git a/docs/03-github-orchestrator/07-advanced-topics/05-configuration-override.mdx b/docs/03-github-orchestrator/07-advanced-topics/05-configuration-override.mdx deleted file mode 100644 index e2ac2640..00000000 --- a/docs/03-github-orchestrator/07-advanced-topics/05-configuration-override.mdx +++ /dev/null @@ -1,42 +0,0 @@ -# Configuration Override - -Pull parameter values from external sources like secret managers or files at runtime. This avoids -hardcoding credentials and keeps CLI commands short. - -``` - Orchestrator External Source - ┌──────────────┐ command ┌──────────────────┐ - │ Reads input │ ──────────────► │ Secret Manager │ - │ override │ │ (GCP, AWS, file) │ - │ list │ ◄────────────── │ │ - │ │ value │ │ - └──────────────┘ └──────────────────┘ -``` - -## Parameters - -| Parameter | Description | -| --------------------------- | ------------------------------------------------------------------ | -| `populateOverride` | Must be `true` to enable override (CLI only). | -| `readInputFromOverrideList` | Comma-separated list of parameter names to pull. | -| `readInputOverrideCommand` | Command to run. Use `{0}` as a placeholder for the parameter name. | - -### Built-in Presets - -Instead of writing a full command, you can use these presets as the `readInputOverrideCommand`: - -| Preset | Expands to | -| -------------------- | ----------------------------------------------------- | -| `gcp-secret-manager` | `gcloud secrets versions access 1 --secret="{0}"` | -| `aws-secret-manager` | `aws secretsmanager get-secret-value --secret-id {0}` | - -## Example - -```bash -yarn run cli -m cli-build \ - --populateOverride true \ - --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD \ - --readInputOverrideCommand='gcloud secrets versions access 1 --secret="{0}"' -``` - -This runs the GCP command for each parameter name in the list and uses the output as the value. diff --git a/docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx b/docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx index 843d578b..8794ea02 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx @@ -20,7 +20,7 @@ Uses the native Kubernetes logging API to stream pod logs directly. Logs flow through a CloudWatch → Kinesis pipeline: -1. Fargate tasks write to **CloudWatch Logs** +1. Orchestrator job (Fargate task) writes logs to **CloudWatch Logs** 2. A **Kinesis** subscription forwards logs in real time 3. Orchestrator consumes from the Kinesis stream From 6f9667fdbb7888cff5179e37537e8000922b6ecd Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:11:48 +0000 Subject: [PATCH 06/20] Fix broken link: hooks directory has no index page Link to container-hooks page instead of the hooks directory. Co-Authored-By: Claude Opus 4.6 --- docs/03-github-orchestrator/01-introduction.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/03-github-orchestrator/01-introduction.mdx b/docs/03-github-orchestrator/01-introduction.mdx index bbc87337..ff5aea21 100644 --- a/docs/03-github-orchestrator/01-introduction.mdx +++ b/docs/03-github-orchestrator/01-introduction.mdx @@ -31,7 +31,7 @@ AWS Fargate and Kubernetes. 1. **Flexible and elastic** — balance speed and cost, configure CPU, memory, and disk per build 2. **Scale from zero** — no idle servers, pay only while builds run 3. **Easy setup** — minimal configuration to [get started](getting-started) -4. **Extensible** — run [custom hooks](advanced-topics/hooks), or bring your own +4. **Extensible** — run [custom hooks](advanced-topics/hooks/container-hooks), or bring your own [provider plugin](providers/custom-providers) ## ❌ When You Don't Need It From 843fe49fb80f0e02d8e6a462b8d9ef27f70c37a1 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:18:06 +0000 Subject: [PATCH 07/20] Add comprehensive GitHub Actions examples page Complete workflow examples for every provider and common patterns: - Minimal workflow, AWS Fargate, Kubernetes, Local Docker - Async mode with GitHub Checks - Scheduled garbage collection - Multi-platform matrix builds - Retained workspaces for faster rebuilds - Container hooks (S3 upload + Steam deploy) - Required secrets tables and cross-links to all relevant docs Co-Authored-By: Claude Opus 4.6 --- .../03-examples/02-github-actions.mdx | 338 ++++++++++++++++++ 1 file changed, 338 insertions(+) create mode 100644 docs/03-github-orchestrator/03-examples/02-github-actions.mdx diff --git a/docs/03-github-orchestrator/03-examples/02-github-actions.mdx b/docs/03-github-orchestrator/03-examples/02-github-actions.mdx new file mode 100644 index 00000000..cc17ad6b --- /dev/null +++ b/docs/03-github-orchestrator/03-examples/02-github-actions.mdx @@ -0,0 +1,338 @@ +# GitHub Actions + +Orchestrator has first-class GitHub Actions support. This page shows complete, copy-paste workflow +files for every provider. + +## 🔑 Prerequisites + +1. A Unity project in a GitHub repository +2. Provider credentials stored as + [GitHub Actions secrets](https://docs.github.com/en/actions/security/encrypted-secrets) +3. A `UNITY_LICENSE` or activation secret (see the + [Game CI activation docs](https://game.ci/docs/github/activation)) + +## Minimal Workflow + +The simplest possible Orchestrator workflow. Uses AWS Fargate with default CPU and memory. + +```yaml +name: Build with Orchestrator + +on: + push: + branches: [main] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-2 + + - uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +## ☁️ AWS Fargate + +Full workflow with custom CPU/memory, S3 artifact export, and GitHub Checks. + +```yaml +name: Orchestrator — AWS Fargate + +on: + push: + branches: [main, develop] + pull_request: + branches: [main] + +jobs: + build: + name: Build (${{ matrix.targetPlatform }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + targetPlatform: + - StandaloneLinux64 + - StandaloneWindows64 + - StandaloneOSX + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-2 + + - uses: game-ci/unity-builder@v4 + id: build + with: + providerStrategy: aws + targetPlatform: ${{ matrix.targetPlatform }} + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} + unityVersion: 2022.3.0f1 + containerCpu: 2048 + containerMemory: 8192 + # Export build artifacts to S3: + containerHookFiles: aws-s3-upload-build + # Show build progress as GitHub Checks: + githubCheck: true +``` + +### Required Secrets + +| Secret | Description | +| ----------------------- | ---------------------------------------------------------------- | +| `AWS_ACCESS_KEY_ID` | IAM access key with ECS, CloudFormation, S3, Kinesis, CloudWatch | +| `AWS_SECRET_ACCESS_KEY` | IAM secret key | + +See the [AWS provider page](../providers/aws) for allowed CPU/memory combinations and full setup. + +## ☸️ Kubernetes + +Full workflow targeting a Kubernetes cluster. + +```yaml +name: Orchestrator — Kubernetes + +on: + push: + branches: [main] + +jobs: + build: + name: Build (${{ matrix.targetPlatform }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + targetPlatform: + - StandaloneLinux64 + - StandaloneWindows64 + env: + kubeConfig: ${{ secrets.KUBE_CONFIG }} + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + - uses: game-ci/unity-builder@v4 + id: build + with: + providerStrategy: k8s + targetPlatform: ${{ matrix.targetPlatform }} + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} + unityVersion: 2022.3.0f1 + kubeVolumeSize: 10Gi + containerCpu: 1024 + containerMemory: 4096 + containerHookFiles: aws-s3-upload-build + githubCheck: true +``` + +### Required Secrets + +| Secret | Description | +| ------------- | ------------------------------- | +| `KUBE_CONFIG` | Base64-encoded kubeconfig file. | + +Generate it with: + +```bash +cat ~/.kube/config | base64 -w 0 +``` + +See the [Kubernetes provider page](../providers/kubernetes) for cluster tips and full setup. + +## 🐳 Local Docker (Self-Hosted Runner) + +Run builds in Docker on your own machine. No cloud account needed. + +```yaml +name: Orchestrator — Local Docker + +on: + push: + branches: [main] + +jobs: + build: + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + - uses: game-ci/unity-builder@v4 + with: + providerStrategy: local-docker + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +Requires Docker installed on the self-hosted runner. + +## ⏳ Async Mode + +For long builds, use async mode so the GitHub Action returns immediately. Monitor progress via +GitHub Checks. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} + asyncOrchestrator: true + githubCheck: true +``` + +The build runs in the background. Check progress from the **Checks** tab on your pull request. + +See [GitHub Integration](../providers/github-integration) for more on async mode and GitHub Checks. + +## 🗑️ Scheduled Garbage Collection + +Add a scheduled workflow to clean up stale cloud resources. Useful as a safety net alongside the +automatic cleanup cron. + +```yaml +name: Orchestrator — Garbage Collect + +on: + schedule: + - cron: '0 4 * * *' # Daily at 4 AM UTC + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-2 + + - uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + mode: garbage-collect + garbageMaxAge: 24 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +See [Garbage Collection](../advanced-topics/garbage-collection) for details. + +## 📦 Multi-Platform Matrix Build + +Build for multiple platforms in parallel. Each platform runs as a separate Orchestrator job. + +```yaml +name: Orchestrator — Multi-Platform + +on: + push: + branches: [main] + +jobs: + build: + name: Build ${{ matrix.targetPlatform }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + targetPlatform: + - StandaloneLinux64 + - StandaloneWindows64 + - StandaloneOSX + - iOS + - Android + - WebGL + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-2 + + - uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + targetPlatform: ${{ matrix.targetPlatform }} + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} + containerCpu: 2048 + containerMemory: 8192 + containerHookFiles: aws-s3-upload-build + githubCheck: true +``` + +## 🔁 Retained Workspaces for Faster Rebuilds + +For large projects, keep the entire project folder cached between builds. Dramatically speeds up +rebuilds at the cost of more storage. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} + maxRetainedWorkspaces: 3 + containerCpu: 2048 + containerMemory: 8192 +``` + +See [Retained Workspaces](../advanced-topics/retained-workspace) and +[Caching](../advanced-topics/caching) for details on storage strategies. + +## 🪝 Container Hooks — S3 Upload + Steam Deploy + +Chain multiple container hooks to export builds to S3 and deploy to Steam in a single workflow. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} + containerHookFiles: aws-s3-upload-build,steam-deploy-client + env: + STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }} + STEAM_PASSWORD: ${{ secrets.STEAM_PASSWORD }} + STEAM_APPID: ${{ secrets.STEAM_APPID }} +``` + +See [Premade Container Hooks](../advanced-topics/hooks/premade-container-jobs) for all available +hooks (S3, rclone, Steam). + +## 🔗 Reference + +- [API Reference](../api-reference) — full list of all parameters +- [Providers](../providers/overview) — setup guides for each provider +- [Secrets](../secrets) — how credentials are transferred to build containers +- [Real-world pipeline](https://github.com/game-ci/unity-builder/blob/main/.github/workflows/orchestrator-integrity.yml) + — Game CI's own Orchestrator test pipeline From e9abe7767438a81ae20f25c3ab5685b225bffcc9 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:21:20 +0000 Subject: [PATCH 08/20] Fix ASCII diagram alignment in Game-CI vs Orchestrator Equalize box widths and arrow spacing for consistent rendering. Co-Authored-By: Claude Opus 4.6 --- .../02-game-ci-vs-orchestrator.mdx | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx b/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx index 59ed793a..e4ce3c61 100644 --- a/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx +++ b/docs/03-github-orchestrator/02-game-ci-vs-orchestrator.mdx @@ -21,17 +21,17 @@ the CI runner itself. This is useful when: - You want to **scale to many concurrent builds** without managing servers ``` - Standard Game-CI Orchestrator Mode - - ┌──────────────┐ ┌──────────────┐ ┌────────────┐ - │ GitHub │ │ GitHub Action │ │ │ - │ Runner │ │ CLI / Any CI │ ───► │ Cloud │ - │ (builds │ │ │ │ Container │ - │ locally) │ │ (dispatches │ ◄─── │ (builds │ - │ │ │ only) │ │ remotely) │ - └──────────────┘ └──────────────┘ └────────────┘ - ~14 GB disk Configurable CPU, memory, disk - Fixed resources Scales to zero when idle + Standard Game-CI Orchestrator Mode + + ┌────────────────┐ ┌─────────────────┐ ┌──────────────┐ + │ GitHub │ │ GitHub Action │ │ │ + │ Runner │ │ CLI / Any CI │───►│ Cloud │ + │ │ │ │ │ Container │ + │ (builds │ │ (dispatches │◄───│ (builds │ + │ locally) │ │ only) │ │ remotely) │ + └────────────────┘ └─────────────────┘ └──────────────┘ + ~14 GB disk Configurable CPU, memory, disk + Fixed resources Scales to zero when idle ``` ## Self-Hosted Runners vs Orchestrator From d1284150033178d6e31d8806ef0461fabf613c8e Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:23:08 +0000 Subject: [PATCH 09/20] Add ASCII diagrams to custom providers, GitHub integration, and retained workspaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Custom Providers: plugin loading flow (source → fetch → ProviderInterface) - GitHub Integration: async mode lifecycle (dispatch → return → Check updates) - Retained Workspaces: workspace locking across concurrent builds Co-Authored-By: Claude Opus 4.6 --- .../05-providers/06-custom-providers.mdx | 13 +++++++++++++ .../05-providers/08-github-integration.mdx | 12 ++++++++++++ .../07-advanced-topics/02-retained-workspace.mdx | 14 ++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/docs/03-github-orchestrator/05-providers/06-custom-providers.mdx b/docs/03-github-orchestrator/05-providers/06-custom-providers.mdx index b855a223..dcb82cba 100644 --- a/docs/03-github-orchestrator/05-providers/06-custom-providers.mdx +++ b/docs/03-github-orchestrator/05-providers/06-custom-providers.mdx @@ -7,6 +7,19 @@ pluggable backend that controls where and how your builds run. Built-in provider With custom providers, you can point `providerStrategy` at a GitHub repository, NPM package, or local path and Orchestrator will dynamically load it at runtime. +``` + providerStrategy Build + ┌─────────────────┐ fetch ┌───────────────┐ ┌──────────────┐ + │ "user/repo" │────────────►│ Clone repo / │ │ │ + │ "npm-package" │ │ Install pkg / │───►│ Provider │ + │ "./local/path" │ │ Resolve path │ │ Interface │ + └─────────────────┘ └───────────────┘ │ │ + cached in │ setupWorkflow│ + .provider-cache/ │ runTask │ + │ cleanup │ + └──────────────┘ +``` + ## Using a Custom Provider Set `providerStrategy` to a provider source instead of a built-in name: diff --git a/docs/03-github-orchestrator/05-providers/08-github-integration.mdx b/docs/03-github-orchestrator/05-providers/08-github-integration.mdx index ca43528c..b9886599 100644 --- a/docs/03-github-orchestrator/05-providers/08-github-integration.mdx +++ b/docs/03-github-orchestrator/05-providers/08-github-integration.mdx @@ -28,6 +28,18 @@ Set [`asyncOrchestrator: true`](../api-reference#github-integration) to start a waiting for it to complete. The GitHub Action will return immediately and you can check progress via GitHub Checks or by running the [`watch` mode](../api-reference#modes). +``` + GitHub Action Cloud GitHub PR + ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ + │ 1. Dispatch │───────────►│ 2. Building │ │ │ + │ 3. Return ✓ │ │ ... │─────────►│ 4. Check │ + │ (done) │ │ ... │ update │ updated │ + └──────────────┘ │ 5. Complete │─────────►│ 6. Check ✓ │ + Action finishes └──────────────┘ └──────────────┘ + in seconds Build runs for Monitor from + minutes/hours the PR page +``` + ```yaml - uses: game-ci/unity-builder@v4 with: diff --git a/docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx b/docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx index ab998997..6faa0023 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx @@ -16,6 +16,20 @@ Each retained workspace is locked during use — only one build can use a worksp Orchestrator handles locking automatically via S3 or rclone. See [Caching](caching) for storage provider details. +``` + maxRetainedWorkspaces: 3 + + Workspace 1 Workspace 2 Workspace 3 + ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ + │ 🔒 Build A │ │ 🔒 Build B │ │ (idle) │ + │ Full project │ │ Full project │ │ Full project │ + │ copy │ │ copy │ │ copy │ + └──────────────┘ └──────────────┘ └──────────────┘ + + Build C arrives → claims Workspace 3 + Build D arrives → all locked → falls back to standard caching +``` + ## Example ```yaml From 759a16f9970ea45d60f460a277a6bc6654ff7b3c Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:25:34 +0000 Subject: [PATCH 10/20] Add ASCII diagrams to container hooks, garbage collection, AWS, and providers overview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Container Hooks: build pipeline with pre/post hook execution points - Garbage Collection: resource lifecycle (normal cleanup vs stale → GC) - AWS: CloudFormation resource stack (ECS, S3, CloudWatch, Kinesis) - Providers Overview: decision flowchart for choosing a provider Co-Authored-By: Claude Opus 4.6 --- .../05-providers/01-overview.mdx | 16 +++++++++++++ .../05-providers/02-aws.mdx | 23 +++++++++++++++++++ .../03-garbage-collection.mdx | 17 ++++++++++++++ .../04-hooks/04-container-hooks.mdx | 12 ++++++++++ 4 files changed, 68 insertions(+) diff --git a/docs/03-github-orchestrator/05-providers/01-overview.mdx b/docs/03-github-orchestrator/05-providers/01-overview.mdx index a5094b96..78e4f1dd 100644 --- a/docs/03-github-orchestrator/05-providers/01-overview.mdx +++ b/docs/03-github-orchestrator/05-providers/01-overview.mdx @@ -3,6 +3,22 @@ A **provider** is the backend that Orchestrator uses to run your builds. You choose a provider by setting the `providerStrategy` parameter. +``` + Which provider should I use? + + Need cloud scaling? ──── Yes ──► Have AWS account? ── Yes ──► aws + │ │ + No No + │ │ + ▼ ▼ + Have Docker? ─── Yes ──► local-docker Have K8s cluster? ── Yes ──► k8s + │ │ + No No + │ │ + ▼ ▼ + local Consider aws (easiest cloud setup) +``` + ## Built-in Providers These providers ship with Orchestrator and are maintained by the Game CI team. diff --git a/docs/03-github-orchestrator/05-providers/02-aws.mdx b/docs/03-github-orchestrator/05-providers/02-aws.mdx index 21ce26dc..38a30c6c 100644 --- a/docs/03-github-orchestrator/05-providers/02-aws.mdx +++ b/docs/03-github-orchestrator/05-providers/02-aws.mdx @@ -1,5 +1,28 @@ # AWS +## Architecture + +Orchestrator creates and manages these AWS resources automatically: + +``` + CloudFormation (base stack) + ┌─────────────────────────────────────────────────┐ + │ │ + │ ┌────────────┐ ┌────────────┐ │ + │ │ ECS │ │ S3 │ │ + │ │ Fargate │ │ Bucket │ │ + │ │ (build │ │ (cache + │ │ + │ │ tasks) │ │ artifacts)│ │ + │ └────────────┘ └────────────┘ │ + │ │ + │ ┌────────────┐ ┌────────────┐ │ + │ │ CloudWatch │───►│ Kinesis │──► Log stream │ + │ │ Logs │ │ Stream │ to CI │ + │ └────────────┘ └────────────┘ │ + │ │ + └─────────────────────────────────────────────────┘ +``` + ## Requirements - An AWS account with permission to create resources (ECS, CloudFormation, S3, Kinesis, CloudWatch). diff --git a/docs/03-github-orchestrator/07-advanced-topics/03-garbage-collection.mdx b/docs/03-github-orchestrator/07-advanced-topics/03-garbage-collection.mdx index 140c036a..38bf1cc8 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/03-garbage-collection.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/03-garbage-collection.mdx @@ -3,6 +3,23 @@ Orchestrator creates cloud resources (containers, stacks, volumes) for each build and cleans them up automatically. If a build fails or is interrupted, resources may be left behind. +``` + Normal Build Failed / Interrupted Build + + ┌──────────┐ ┌──────────┐ ┌─────┐ ┌──────────┐ ┌──────────┐ + │ Create │─►│ Build │─►│ ✓ │ │ Create │─►│ Build │──► ✗ crash + │ resources│ │ │ │Clean│ │ resources│ │ │ + └──────────┘ └──────────┘ └─────┘ └──────────┘ └──────────┘ + auto ▼ + cleanup Resources left behind + ▼ + ┌─────────────────┐ + │ garbage-collect │ + │ removes after │ + │ garbageMaxAge │ + └─────────────────┘ +``` + Use garbage collection to clean up stale resources. See the [API Reference](../api-reference#garbage-collection) for all parameters. diff --git a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx index 0931539c..61f55d0e 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx @@ -4,6 +4,18 @@ Run custom Docker containers as steps in the build workflow. Useful for uploadin deploying builds, or running additional tools. For inline shell commands instead, see [Command Hooks](command-hooks). +``` + Build Pipeline + + preBuildContainerHooks Unity Build postBuildContainerHooks + ┌─────────────────────┐ ┌──────────────┐ ┌──────────────────────┐ + │ 🐳 Pull cache │ │ │ │ 🐳 Upload build │ + │ 🐳 Setup deps │───►│ 🔨 Build │───►│ 🐳 Deploy to Steam │ + │ 🐳 ... │ │ │ │ 🐳 ... │ + └─────────────────────┘ └──────────────┘ └──────────────────────┘ + Runs before build Core build step Runs after build +``` + ## Format ```yaml From db90cbc3cc3469ae4eaf279ca6be38dfae90b745 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:26:19 +0000 Subject: [PATCH 11/20] Replace provider decision tree with simple 4-across comparison Shows each provider side-by-side with its key trait instead of a decision flowchart. Co-Authored-By: Claude Opus 4.6 --- .../05-providers/01-overview.mdx | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/docs/03-github-orchestrator/05-providers/01-overview.mdx b/docs/03-github-orchestrator/05-providers/01-overview.mdx index 78e4f1dd..ff91331c 100644 --- a/docs/03-github-orchestrator/05-providers/01-overview.mdx +++ b/docs/03-github-orchestrator/05-providers/01-overview.mdx @@ -4,19 +4,15 @@ A **provider** is the backend that Orchestrator uses to run your builds. You cho setting the `providerStrategy` parameter. ``` - Which provider should I use? - - Need cloud scaling? ──── Yes ──► Have AWS account? ── Yes ──► aws - │ │ - No No - │ │ - ▼ ▼ - Have Docker? ─── Yes ──► local-docker Have K8s cluster? ── Yes ──► k8s - │ │ - No No - │ │ - ▼ ▼ - local Consider aws (easiest cloud setup) + ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ + │ aws │ │ k8s │ │ local-docker │ │ local │ + │ │ │ │ │ │ │ │ + │ ☁️ Fargate │ │ ☸️ Cluster │ │ 🐳 Container │ │ 🖥️ Direct │ + │ Fully │ │ Bring your │ │ No cloud │ │ No container │ + │ managed │ │ own cluster │ │ needed │ │ needed │ + └──────────────┘ └──────────────┘ └──────────────┘ └──────────────┘ + Cloud scaling Cloud scaling Local builds Local builds + No servers Flexible Docker required Simplest setup ``` ## Built-in Providers From 07da84a39f17ce7253f2233861b89f187e35f2e9 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:27:07 +0000 Subject: [PATCH 12/20] Fix sidebar ordering: Secrets before Advanced Topics Set Advanced Topics position to 7.0 so it renders after Secrets (position 6 from filename). Co-Authored-By: Claude Opus 4.6 --- docs/03-github-orchestrator/07-advanced-topics/_category_.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/03-github-orchestrator/07-advanced-topics/_category_.yaml b/docs/03-github-orchestrator/07-advanced-topics/_category_.yaml index da411968..24f7ad99 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/_category_.yaml +++ b/docs/03-github-orchestrator/07-advanced-topics/_category_.yaml @@ -1,5 +1,5 @@ --- -position: 5.0 +position: 7.0 label: Advanced topics collapsible: true collapsed: true From 557659defa6009890e3d7b4c6c829d181b235765 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:28:35 +0000 Subject: [PATCH 13/20] Rename Premade Container Hooks to Built-In Hooks Update title and all cross-references across container hooks, command hooks, and GitHub Actions examples. Co-Authored-By: Claude Opus 4.6 --- docs/03-github-orchestrator/03-examples/02-github-actions.mdx | 4 ++-- .../07-advanced-topics/04-hooks/03-command-hooks.mdx | 2 +- .../07-advanced-topics/04-hooks/04-container-hooks.mdx | 2 +- .../07-advanced-topics/04-hooks/05-premade-container-jobs.mdx | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/03-github-orchestrator/03-examples/02-github-actions.mdx b/docs/03-github-orchestrator/03-examples/02-github-actions.mdx index cc17ad6b..ecd1b30e 100644 --- a/docs/03-github-orchestrator/03-examples/02-github-actions.mdx +++ b/docs/03-github-orchestrator/03-examples/02-github-actions.mdx @@ -326,8 +326,8 @@ Chain multiple container hooks to export builds to S3 and deploy to Steam in a s STEAM_APPID: ${{ secrets.STEAM_APPID }} ``` -See [Premade Container Hooks](../advanced-topics/hooks/premade-container-jobs) for all available -hooks (S3, rclone, Steam). +See [Built-In Hooks](../advanced-topics/hooks/premade-container-jobs) for all available hooks (S3, +rclone, Steam). ## 🔗 Reference diff --git a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/03-command-hooks.mdx b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/03-command-hooks.mdx index 918420f6..14c58490 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/03-command-hooks.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/03-command-hooks.mdx @@ -37,4 +37,4 @@ directory via `customHookFiles`. Place hook files at `.game-ci/hooks/my-hook.yaml` in your repository. For running Docker containers as build steps, see [Container Hooks](container-hooks). For -ready-to-use hooks (S3, rclone, Steam), see [Premade Container Hooks](premade-container-jobs). +ready-to-use hooks (S3, rclone, Steam), see [Built-In Hooks](premade-container-jobs). diff --git a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx index 61f55d0e..9bc19015 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx @@ -42,4 +42,4 @@ files from `.game-ci/container-hooks/` via `containerHookFiles`. ## Pre-built Hooks Orchestrator ships with ready-to-use hooks for S3, rclone, and Steam. See -[Premade Container Hooks](premade-container-jobs) for the full list. +[Built-In Hooks](premade-container-jobs) for the full list. diff --git a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/05-premade-container-jobs.mdx b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/05-premade-container-jobs.mdx index 48a61d01..40b53c80 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/05-premade-container-jobs.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/04-hooks/05-premade-container-jobs.mdx @@ -1,4 +1,4 @@ -# Premade Container Hooks +# Built-In Hooks Orchestrator ships with pre-built container hooks for common tasks. Use them by name with the `containerHookFiles` parameter. From 9df05bccf730cd3a9760c45de11cd95edd6a967c Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:30:59 +0000 Subject: [PATCH 14/20] Rename Built-In Hooks file, move Custom Job out of Hooks, fix alignment - Rename premade-container-jobs.mdx to built-in-hooks.mdx (fixes URL slug) - Update all links from premade-container-jobs to built-in-hooks - Rename "Pre-built Hooks" section header to "Built-In Hooks" - Move Custom Job from hooks/ to advanced-topics/ (it's not a hook) - Rename "Custom Jobs" to "Custom Job" (singular) - Update API reference link to advanced-topics/custom-job - Fix numbering conflicts in advanced-topics - Fix retained workspace diagram alignment (remove emoji, align box walls) Co-Authored-By: Claude Opus 4.6 --- .../03-examples/02-github-actions.mdx | 2 +- .../04-api-reference.mdx | 2 +- .../02-retained-workspace.mdx | 18 +++++++++--------- .../01-custom-job.mdx => 03-custom-job.mdx} | 2 +- ...ollection.mdx => 04-garbage-collection.mdx} | 0 .../03-command-hooks.mdx | 2 +- .../04-container-hooks.mdx | 4 ++-- .../05-built-in-hooks.mdx} | 0 .../{04-hooks => 05-hooks}/_category_.yaml | 0 9 files changed, 15 insertions(+), 15 deletions(-) rename docs/03-github-orchestrator/07-advanced-topics/{04-hooks/01-custom-job.mdx => 03-custom-job.mdx} (97%) rename docs/03-github-orchestrator/07-advanced-topics/{03-garbage-collection.mdx => 04-garbage-collection.mdx} (100%) rename docs/03-github-orchestrator/07-advanced-topics/{04-hooks => 05-hooks}/03-command-hooks.mdx (91%) rename docs/03-github-orchestrator/07-advanced-topics/{04-hooks => 05-hooks}/04-container-hooks.mdx (95%) rename docs/03-github-orchestrator/07-advanced-topics/{04-hooks/05-premade-container-jobs.mdx => 05-hooks/05-built-in-hooks.mdx} (100%) rename docs/03-github-orchestrator/07-advanced-topics/{04-hooks => 05-hooks}/_category_.yaml (100%) diff --git a/docs/03-github-orchestrator/03-examples/02-github-actions.mdx b/docs/03-github-orchestrator/03-examples/02-github-actions.mdx index ecd1b30e..34e1d7dd 100644 --- a/docs/03-github-orchestrator/03-examples/02-github-actions.mdx +++ b/docs/03-github-orchestrator/03-examples/02-github-actions.mdx @@ -326,7 +326,7 @@ Chain multiple container hooks to export builds to S3 and deploy to Steam in a s STEAM_APPID: ${{ secrets.STEAM_APPID }} ``` -See [Built-In Hooks](../advanced-topics/hooks/premade-container-jobs) for all available hooks (S3, +See [Built-In Hooks](../advanced-topics/hooks/built-in-hooks) for all available hooks (S3, rclone, Steam). ## 🔗 Reference diff --git a/docs/03-github-orchestrator/04-api-reference.mdx b/docs/03-github-orchestrator/04-api-reference.mdx index 3e72974a..38824bc0 100644 --- a/docs/03-github-orchestrator/04-api-reference.mdx +++ b/docs/03-github-orchestrator/04-api-reference.mdx @@ -61,7 +61,7 @@ Set the mode to control what Orchestrator does. Default: `cli-build`. | `preBuildSteps` | — | Pre-build job (after repo setup, before build) in YAML format. | | `postBuildContainerHooks` | — | Container hook files to run after the build step. | | `preBuildContainerHooks` | — | Container hook files to run before the build step. | -| `customJob` | — | Custom job YAML to override the default build workflow. See [Custom Job](advanced-topics/hooks/custom-job). | +| `customJob` | — | Custom job YAML to override the default build workflow. See [Custom Job](advanced-topics/custom-job). | ### Pull Secrets diff --git a/docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx b/docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx index 6faa0023..f4076e34 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/02-retained-workspace.mdx @@ -19,15 +19,15 @@ provider details. ``` maxRetainedWorkspaces: 3 - Workspace 1 Workspace 2 Workspace 3 - ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ - │ 🔒 Build A │ │ 🔒 Build B │ │ (idle) │ - │ Full project │ │ Full project │ │ Full project │ - │ copy │ │ copy │ │ copy │ - └──────────────┘ └──────────────┘ └──────────────┘ - - Build C arrives → claims Workspace 3 - Build D arrives → all locked → falls back to standard caching + Workspace 1 Workspace 2 Workspace 3 + ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ + │ [locked] │ │ [locked] │ │ (idle) │ + │ Build A │ │ Build B │ │ │ + │ Full project │ │ Full project │ │ Full project │ + └───────────────┘ └───────────────┘ └───────────────┘ + + Build C arrives --> claims Workspace 3 + Build D arrives --> all locked --> falls back to standard caching ``` ## Example diff --git a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/01-custom-job.mdx b/docs/03-github-orchestrator/07-advanced-topics/03-custom-job.mdx similarity index 97% rename from docs/03-github-orchestrator/07-advanced-topics/04-hooks/01-custom-job.mdx rename to docs/03-github-orchestrator/07-advanced-topics/03-custom-job.mdx index 22d35a90..4bcaddc8 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/01-custom-job.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/03-custom-job.mdx @@ -1,4 +1,4 @@ -# Custom Jobs +# Custom Job Override the default build workflow entirely by specifying the `customJob` parameter with a YAML job definition. diff --git a/docs/03-github-orchestrator/07-advanced-topics/03-garbage-collection.mdx b/docs/03-github-orchestrator/07-advanced-topics/04-garbage-collection.mdx similarity index 100% rename from docs/03-github-orchestrator/07-advanced-topics/03-garbage-collection.mdx rename to docs/03-github-orchestrator/07-advanced-topics/04-garbage-collection.mdx diff --git a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/03-command-hooks.mdx b/docs/03-github-orchestrator/07-advanced-topics/05-hooks/03-command-hooks.mdx similarity index 91% rename from docs/03-github-orchestrator/07-advanced-topics/04-hooks/03-command-hooks.mdx rename to docs/03-github-orchestrator/07-advanced-topics/05-hooks/03-command-hooks.mdx index 14c58490..5e0a71cc 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/03-command-hooks.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/05-hooks/03-command-hooks.mdx @@ -37,4 +37,4 @@ directory via `customHookFiles`. Place hook files at `.game-ci/hooks/my-hook.yaml` in your repository. For running Docker containers as build steps, see [Container Hooks](container-hooks). For -ready-to-use hooks (S3, rclone, Steam), see [Built-In Hooks](premade-container-jobs). +ready-to-use hooks (S3, rclone, Steam), see [Built-In Hooks](built-in-hooks). diff --git a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx b/docs/03-github-orchestrator/07-advanced-topics/05-hooks/04-container-hooks.mdx similarity index 95% rename from docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx rename to docs/03-github-orchestrator/07-advanced-topics/05-hooks/04-container-hooks.mdx index 9bc19015..70a710e5 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/04-container-hooks.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/05-hooks/04-container-hooks.mdx @@ -39,7 +39,7 @@ files from `.game-ci/container-hooks/` via `containerHookFiles`. gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} ``` -## Pre-built Hooks +## Built-In Hooks Orchestrator ships with ready-to-use hooks for S3, rclone, and Steam. See -[Built-In Hooks](premade-container-jobs) for the full list. +[Built-In Hooks](built-in-hooks) for the full list. diff --git a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/05-premade-container-jobs.mdx b/docs/03-github-orchestrator/07-advanced-topics/05-hooks/05-built-in-hooks.mdx similarity index 100% rename from docs/03-github-orchestrator/07-advanced-topics/04-hooks/05-premade-container-jobs.mdx rename to docs/03-github-orchestrator/07-advanced-topics/05-hooks/05-built-in-hooks.mdx diff --git a/docs/03-github-orchestrator/07-advanced-topics/04-hooks/_category_.yaml b/docs/03-github-orchestrator/07-advanced-topics/05-hooks/_category_.yaml similarity index 100% rename from docs/03-github-orchestrator/07-advanced-topics/04-hooks/_category_.yaml rename to docs/03-github-orchestrator/07-advanced-topics/05-hooks/_category_.yaml From b70b662abe5a9a91bec8f7be61decf227f6732a6 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:35:58 +0000 Subject: [PATCH 15/20] Fix alignment of all ASCII diagrams across orchestrator docs Remove emoji characters from diagrams (variable width across platforms makes alignment impossible). Fix box wall alignment, arrow connections, and consistent spacing in all 11 diagrams: - Introduction (architecture overview) - Caching (standard vs retained) - Providers overview (4-across comparison) - Container hooks (build pipeline) - GitHub integration (async mode lifecycle) - AWS (CloudFormation resource stack) - Secrets (pull flow) - Logging (log pipeline) - Garbage collection (resource lifecycle) - Custom providers (plugin loading) - Retained workspaces (already fixed) Co-Authored-By: Claude Opus 4.6 --- .../01-introduction.mdx | 28 ++++++++--------- .../03-examples/02-github-actions.mdx | 4 +-- .../04-api-reference.mdx | 20 ++++++------- .../05-providers/01-overview.mdx | 2 +- .../05-providers/02-aws.mdx | 30 +++++++++---------- .../05-providers/06-custom-providers.mdx | 20 ++++++------- .../05-providers/08-github-integration.mdx | 18 +++++------ docs/03-github-orchestrator/06-secrets.mdx | 14 ++++----- .../07-advanced-topics/01-caching.mdx | 18 +++++------ .../04-garbage-collection.mdx | 30 ++++++++++--------- .../05-hooks/04-container-hooks.mdx | 14 ++++----- .../07-advanced-topics/06-logging.mdx | 10 +++---- 12 files changed, 105 insertions(+), 103 deletions(-) diff --git a/docs/03-github-orchestrator/01-introduction.mdx b/docs/03-github-orchestrator/01-introduction.mdx index ff5aea21..1060b6b3 100644 --- a/docs/03-github-orchestrator/01-introduction.mdx +++ b/docs/03-github-orchestrator/01-introduction.mdx @@ -7,20 +7,20 @@ command line, or any CI system. Orchestrator provisions a cloud environment, sen be built, and streams results back. ``` - Your Machine / CI Cloud Provider - ┌──────────────┐ git push ┌──────────────────┐ - │ │ ──────────────────► │ ☁️ AWS Fargate │ - │ GitHub │ │ ☁️ Kubernetes │ - │ Actions │ ◄──────────────── │ 🐳 Local Docker │ - │ │ build artifacts │ │ - └──────────────┘ └──────────────────┘ - │ │ - │ Orchestrator handles: │ - │ • Provisioning │ - │ • Git sync + LFS │ - │ • Caching (S3 / rclone) │ - │ • Automatic cleanup │ - └─────────────────────────────────────┘ + Your Machine / CI Cloud Provider + ┌──────────────┐ git push ┌─────────────────┐ + │ │ ─────────────────►│ AWS Fargate │ + │ GitHub │ │ Kubernetes │ + │ Actions │ ◄─────────────────│ Local Docker │ + │ │ build artifacts │ │ + └──────────────┘ └─────────────────┘ + │ │ + │ Orchestrator handles: │ + │ * Provisioning │ + │ * Git sync + LFS │ + │ * Caching (S3 / rclone) │ + │ * Automatic cleanup │ + └───────────────────────────────────┘ ``` Orchestrator supports large projects with first-class Unity support and native cloud services like diff --git a/docs/03-github-orchestrator/03-examples/02-github-actions.mdx b/docs/03-github-orchestrator/03-examples/02-github-actions.mdx index 34e1d7dd..e069d311 100644 --- a/docs/03-github-orchestrator/03-examples/02-github-actions.mdx +++ b/docs/03-github-orchestrator/03-examples/02-github-actions.mdx @@ -326,8 +326,8 @@ Chain multiple container hooks to export builds to S3 and deploy to Steam in a s STEAM_APPID: ${{ secrets.STEAM_APPID }} ``` -See [Built-In Hooks](../advanced-topics/hooks/built-in-hooks) for all available hooks (S3, -rclone, Steam). +See [Built-In Hooks](../advanced-topics/hooks/built-in-hooks) for all available hooks (S3, rclone, +Steam). ## 🔗 Reference diff --git a/docs/03-github-orchestrator/04-api-reference.mdx b/docs/03-github-orchestrator/04-api-reference.mdx index 38824bc0..4048dd84 100644 --- a/docs/03-github-orchestrator/04-api-reference.mdx +++ b/docs/03-github-orchestrator/04-api-reference.mdx @@ -52,16 +52,16 @@ Set the mode to control what Orchestrator does. Default: `cli-build`. ### Custom Hooks -| Parameter | Default | Description | -| ------------------------- | ------- | ----------------------------------------------------------------------------------------------------------- | -| `containerHookFiles` | — | Names of [container hook](advanced-topics/hooks/container-hooks) files from `.game-ci/container-hooks/`. | -| `customHookFiles` | — | Names of custom hook files from `.game-ci/hooks/`. | -| `customCommandHooks` | — | Inline [command hooks](advanced-topics/hooks/command-hooks) as YAML. | -| `postBuildSteps` | — | Post-build job in YAML format with keys: `image`, `secrets`, `command`. | -| `preBuildSteps` | — | Pre-build job (after repo setup, before build) in YAML format. | -| `postBuildContainerHooks` | — | Container hook files to run after the build step. | -| `preBuildContainerHooks` | — | Container hook files to run before the build step. | -| `customJob` | — | Custom job YAML to override the default build workflow. See [Custom Job](advanced-topics/custom-job). | +| Parameter | Default | Description | +| ------------------------- | ------- | -------------------------------------------------------------------------------------------------------- | +| `containerHookFiles` | — | Names of [container hook](advanced-topics/hooks/container-hooks) files from `.game-ci/container-hooks/`. | +| `customHookFiles` | — | Names of custom hook files from `.game-ci/hooks/`. | +| `customCommandHooks` | — | Inline [command hooks](advanced-topics/hooks/command-hooks) as YAML. | +| `postBuildSteps` | — | Post-build job in YAML format with keys: `image`, `secrets`, `command`. | +| `preBuildSteps` | — | Pre-build job (after repo setup, before build) in YAML format. | +| `postBuildContainerHooks` | — | Container hook files to run after the build step. | +| `preBuildContainerHooks` | — | Container hook files to run before the build step. | +| `customJob` | — | Custom job YAML to override the default build workflow. See [Custom Job](advanced-topics/custom-job). | ### Pull Secrets diff --git a/docs/03-github-orchestrator/05-providers/01-overview.mdx b/docs/03-github-orchestrator/05-providers/01-overview.mdx index ff91331c..be952d2d 100644 --- a/docs/03-github-orchestrator/05-providers/01-overview.mdx +++ b/docs/03-github-orchestrator/05-providers/01-overview.mdx @@ -7,7 +7,7 @@ setting the `providerStrategy` parameter. ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ aws │ │ k8s │ │ local-docker │ │ local │ │ │ │ │ │ │ │ │ - │ ☁️ Fargate │ │ ☸️ Cluster │ │ 🐳 Container │ │ 🖥️ Direct │ + │ Fargate │ │ Cluster │ │ Container │ │ Direct │ │ Fully │ │ Bring your │ │ No cloud │ │ No container │ │ managed │ │ own cluster │ │ needed │ │ needed │ └──────────────┘ └──────────────┘ └──────────────┘ └──────────────┘ diff --git a/docs/03-github-orchestrator/05-providers/02-aws.mdx b/docs/03-github-orchestrator/05-providers/02-aws.mdx index 38a30c6c..3beabbe1 100644 --- a/docs/03-github-orchestrator/05-providers/02-aws.mdx +++ b/docs/03-github-orchestrator/05-providers/02-aws.mdx @@ -6,21 +6,21 @@ Orchestrator creates and manages these AWS resources automatically: ``` CloudFormation (base stack) - ┌─────────────────────────────────────────────────┐ - │ │ - │ ┌────────────┐ ┌────────────┐ │ - │ │ ECS │ │ S3 │ │ - │ │ Fargate │ │ Bucket │ │ - │ │ (build │ │ (cache + │ │ - │ │ tasks) │ │ artifacts)│ │ - │ └────────────┘ └────────────┘ │ - │ │ - │ ┌────────────┐ ┌────────────┐ │ - │ │ CloudWatch │───►│ Kinesis │──► Log stream │ - │ │ Logs │ │ Stream │ to CI │ - │ └────────────┘ └────────────┘ │ - │ │ - └─────────────────────────────────────────────────┘ + ┌──────────────────────────────────────────────────┐ + │ │ + │ ┌────────────┐ ┌─────────────┐ │ + │ │ ECS │ │ S3 │ │ + │ │ Fargate │ │ Bucket │ │ + │ │ (build │ │ (cache + │ │ + │ │ tasks) │ │ artifacts) │ │ + │ └────────────┘ └─────────────┘ │ + │ │ + │ ┌────────────┐ ┌─────────────┐ │ + │ │ CloudWatch │───►│ Kinesis │──► Log stream │ + │ │ Logs │ │ Stream │ to CI │ + │ └────────────┘ └─────────────┘ │ + │ │ + └──────────────────────────────────────────────────┘ ``` ## Requirements diff --git a/docs/03-github-orchestrator/05-providers/06-custom-providers.mdx b/docs/03-github-orchestrator/05-providers/06-custom-providers.mdx index dcb82cba..93688b1b 100644 --- a/docs/03-github-orchestrator/05-providers/06-custom-providers.mdx +++ b/docs/03-github-orchestrator/05-providers/06-custom-providers.mdx @@ -8,16 +8,16 @@ With custom providers, you can point `providerStrategy` at a GitHub repository, local path and Orchestrator will dynamically load it at runtime. ``` - providerStrategy Build - ┌─────────────────┐ fetch ┌───────────────┐ ┌──────────────┐ - │ "user/repo" │────────────►│ Clone repo / │ │ │ - │ "npm-package" │ │ Install pkg / │───►│ Provider │ - │ "./local/path" │ │ Resolve path │ │ Interface │ - └─────────────────┘ └───────────────┘ │ │ - cached in │ setupWorkflow│ - .provider-cache/ │ runTask │ - │ cleanup │ - └──────────────┘ + providerStrategy Build + ┌─────────────────┐ fetch ┌────────────────┐ ┌──────────────┐ + │ "user/repo" │──────────►│ Clone repo / │ │ │ + │ "npm-package" │ │ Install pkg / │──►│ Provider │ + │ "./local/path" │ │ Resolve path │ │ Interface │ + └─────────────────┘ └────────────────┘ │ │ + cached in │ setupWorkflow│ + .provider-cache/ │ runTask │ + │ cleanup │ + └──────────────┘ ``` ## Using a Custom Provider diff --git a/docs/03-github-orchestrator/05-providers/08-github-integration.mdx b/docs/03-github-orchestrator/05-providers/08-github-integration.mdx index b9886599..9225960f 100644 --- a/docs/03-github-orchestrator/05-providers/08-github-integration.mdx +++ b/docs/03-github-orchestrator/05-providers/08-github-integration.mdx @@ -29,15 +29,15 @@ waiting for it to complete. The GitHub Action will return immediately and you ca GitHub Checks or by running the [`watch` mode](../api-reference#modes). ``` - GitHub Action Cloud GitHub PR - ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ - │ 1. Dispatch │───────────►│ 2. Building │ │ │ - │ 3. Return ✓ │ │ ... │─────────►│ 4. Check │ - │ (done) │ │ ... │ update │ updated │ - └──────────────┘ │ 5. Complete │─────────►│ 6. Check ✓ │ - Action finishes └──────────────┘ └──────────────┘ - in seconds Build runs for Monitor from - minutes/hours the PR page + GitHub Action Cloud GitHub PR + ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ + │ 1. Dispatch │──────────►│ 2. Building │ │ │ + │ 3. Return │ │ ... │────────►│ 4. Check │ + │ (done) │ │ ... │ update │ updated │ + └──────────────┘ │ 5. Complete │────────►│ 6. Check │ + Action finishes └──────────────┘ └──────────────┘ + in seconds Build runs for Monitor from + minutes/hours the PR page ``` ```yaml diff --git a/docs/03-github-orchestrator/06-secrets.mdx b/docs/03-github-orchestrator/06-secrets.mdx index d79182cb..9354f08d 100644 --- a/docs/03-github-orchestrator/06-secrets.mdx +++ b/docs/03-github-orchestrator/06-secrets.mdx @@ -19,13 +19,13 @@ You can pull parameter values from external secret managers or files at runtime hardcoding credentials. This keeps CLI commands short and secrets out of your repository. ``` - Orchestrator External Source - ┌──────────────┐ command ┌──────────────────┐ - │ Reads input │ ──────────────► │ Secret Manager │ - │ override │ │ (GCP, AWS, file) │ - │ list │ ◄────────────── │ │ - │ │ value │ │ - └──────────────┘ └──────────────────┘ + Orchestrator External Source + ┌──────────────┐ command ┌──────────────────┐ + │ Reads input │ ────────────────►│ Secret Manager │ + │ override │ │ (GCP, AWS, file) │ + │ list │ ◄────────────────│ │ + │ │ value │ │ + └──────────────┘ └──────────────────┘ ``` ### Parameters diff --git a/docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx b/docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx index 40ee1ef6..be546e84 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx @@ -5,15 +5,15 @@ Orchestrator supports two caching strategies. You can mix both by setting fall back to standard caching. ``` - Standard Caching Retained Workspace - ┌─────────────────┐ ┌─────────────────┐ - │ 📁 LFS files │ │ 📁 Entire │ - │ 📁 Library/ │ │ project │ - │ │ │ folder │ - │ Smaller storage │ │ Faster builds │ - │ Good for small │ │ Good for large │ - │ projects │ │ projects │ - └─────────────────┘ └─────────────────┘ + Standard Caching Retained Workspace + ┌─────────────────┐ ┌─────────────────┐ + │ LFS files │ │ Entire project │ + │ Library/ │ │ folder │ + │ │ │ │ + │ Smaller storage │ │ Faster builds │ + │ Good for small │ │ Good for large │ + │ projects │ │ projects │ + └─────────────────┘ └─────────────────┘ ``` ## Standard Caching diff --git a/docs/03-github-orchestrator/07-advanced-topics/04-garbage-collection.mdx b/docs/03-github-orchestrator/07-advanced-topics/04-garbage-collection.mdx index 38bf1cc8..30830fba 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/04-garbage-collection.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/04-garbage-collection.mdx @@ -4,20 +4,22 @@ Orchestrator creates cloud resources (containers, stacks, volumes) for each buil automatically. If a build fails or is interrupted, resources may be left behind. ``` - Normal Build Failed / Interrupted Build - - ┌──────────┐ ┌──────────┐ ┌─────┐ ┌──────────┐ ┌──────────┐ - │ Create │─►│ Build │─►│ ✓ │ │ Create │─►│ Build │──► ✗ crash - │ resources│ │ │ │Clean│ │ resources│ │ │ - └──────────┘ └──────────┘ └─────┘ └──────────┘ └──────────┘ - auto ▼ - cleanup Resources left behind - ▼ - ┌─────────────────┐ - │ garbage-collect │ - │ removes after │ - │ garbageMaxAge │ - └─────────────────┘ + Normal Build Failed / Interrupted Build + + ┌──────────┐ ┌──────────┐ Auto ┌──────────┐ ┌──────────┐ + │ Create │─►│ Build │─►Clean │ Create │─►│ Build │──► crash + │ resources│ │ │ up │ resources│ │ │ + └──────────┘ └──────────┘ └──────────┘ └──────────┘ + | + v + Resources left behind + | + v + ┌─────────────────┐ + │ garbage-collect │ + │ removes after │ + │ garbageMaxAge │ + └─────────────────┘ ``` Use garbage collection to clean up stale resources. See the diff --git a/docs/03-github-orchestrator/07-advanced-topics/05-hooks/04-container-hooks.mdx b/docs/03-github-orchestrator/07-advanced-topics/05-hooks/04-container-hooks.mdx index 70a710e5..f88a41ae 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/05-hooks/04-container-hooks.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/05-hooks/04-container-hooks.mdx @@ -7,13 +7,13 @@ deploying builds, or running additional tools. For inline shell commands instead ``` Build Pipeline - preBuildContainerHooks Unity Build postBuildContainerHooks - ┌─────────────────────┐ ┌──────────────┐ ┌──────────────────────┐ - │ 🐳 Pull cache │ │ │ │ 🐳 Upload build │ - │ 🐳 Setup deps │───►│ 🔨 Build │───►│ 🐳 Deploy to Steam │ - │ 🐳 ... │ │ │ │ 🐳 ... │ - └─────────────────────┘ └──────────────┘ └──────────────────────┘ - Runs before build Core build step Runs after build + preBuildContainerHooks Unity Build postBuildContainerHooks + ┌──────────────────────┐ ┌──────────────┐ ┌──────────────────────┐ + │ Pull cache │ │ │ │ Upload build │ + │ Setup deps │──►│ Build │──►│ Deploy to Steam │ + │ ... │ │ │ │ ... │ + └──────────────────────┘ └──────────────┘ └──────────────────────┘ + Runs before build Core build step Runs after build ``` ## Format diff --git a/docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx b/docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx index 8794ea02..e7c8c9af 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/06-logging.mdx @@ -3,11 +3,11 @@ Orchestrator streams logs from the remote build back to your CI runner in real time. ``` - Cloud Container Orchestrator Your CI - ┌───────────────┐ ┌──────────┐ ┌──────────────┐ - │ Build output │ ─────► │ Log │ ─────► │ Console │ - │ │ │ stream │ │ output │ - └───────────────┘ └──────────┘ └──────────────┘ + Cloud Container Orchestrator Your CI + ┌───────────────┐ ┌──────────────┐ ┌──────────────┐ + │ Build output │──────►│ Log stream │─────►│ Console │ + │ │ │ │ │ output │ + └───────────────┘ └──────────────┘ └──────────────┘ ``` ## Provider-Specific Log Transport From c7ac032886df512c75da403a314bf419c6c017bd Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:47:22 +0000 Subject: [PATCH 16/20] Add Load Balancing documentation page Documents how to route builds across multiple providers using GitHub Actions scripting: platform-based routing, branch-based routing, runner availability fallback, weighted distribution, and async mode integration. Co-Authored-By: Claude Opus 4.6 --- .../07-advanced-topics/07-load-balancing.mdx | 257 ++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 docs/03-github-orchestrator/07-advanced-topics/07-load-balancing.mdx diff --git a/docs/03-github-orchestrator/07-advanced-topics/07-load-balancing.mdx b/docs/03-github-orchestrator/07-advanced-topics/07-load-balancing.mdx new file mode 100644 index 00000000..eb527e54 --- /dev/null +++ b/docs/03-github-orchestrator/07-advanced-topics/07-load-balancing.mdx @@ -0,0 +1,257 @@ +# Load Balancing + +Orchestrator supports multiple providers, and you can use standard GitHub Actions scripting to route +builds across them. This lets you distribute builds based on platform, project size, runner +availability, or any custom logic. + +``` + ┌──────────────────────┐ + │ Incoming Build │ + │ Request │ + └──────────┬─────────────┘ + │ + ┌──────────▼─────────────┐ + │ Router Logic │ + │ (GitHub Actions / │ + │ script / matrix) │ + └──┬───────┬──────────┬───┘ + │ │ │ + ┌──────────────▼┐ ┌───▼────────┐ ┌▼──────────────┐ + │ aws │ │ k8s │ │ local-docker │ + │ │ │ │ │ │ + │ Fargate │ │ Cluster │ │ Self-hosted │ + │ (scalable) │ │ (flexible) │ │ (no cost) │ + └───────────────┘ └────────────┘ └───────────────┘ +``` + +## Why Load Balance? + +- **Cost control** — Route small or test builds to free self-hosted runners and reserve cloud + providers for production builds. +- **Availability** — Fall back to a cloud provider when your self-hosted runner is busy or offline. +- **Platform routing** — Send Linux builds to AWS and Windows builds to a local runner. +- **Parallel scaling** — Split a multi-platform matrix across providers to finish faster. + +## Basic Provider Routing + +Use a GitHub Actions matrix or conditional logic to route builds to different providers. + +### Route by Platform + +```yaml +name: Platform-Based Routing + +on: + push: + branches: [main] + +jobs: + build: + name: Build ${{ matrix.targetPlatform }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # Linux builds go to AWS (fast, scalable) + - targetPlatform: StandaloneLinux64 + provider: aws + # Windows builds go to self-hosted runner + - targetPlatform: StandaloneWindows64 + provider: local-docker + # WebGL builds go to Kubernetes + - targetPlatform: WebGL + provider: k8s + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + - uses: game-ci/unity-builder@v4 + with: + providerStrategy: ${{ matrix.provider }} + targetPlatform: ${{ matrix.targetPlatform }} + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +### Route by Branch + +Send production builds to a high-resource cloud provider and development builds to a cheaper option. + +```yaml +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + - name: Select provider + id: provider + run: | + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + echo "strategy=aws" >> "$GITHUB_OUTPUT" + echo "cpu=4096" >> "$GITHUB_OUTPUT" + echo "memory=16384" >> "$GITHUB_OUTPUT" + else + echo "strategy=local-docker" >> "$GITHUB_OUTPUT" + echo "cpu=1024" >> "$GITHUB_OUTPUT" + echo "memory=3072" >> "$GITHUB_OUTPUT" + fi + + - uses: game-ci/unity-builder@v4 + with: + providerStrategy: ${{ steps.provider.outputs.strategy }} + targetPlatform: StandaloneLinux64 + containerCpu: ${{ steps.provider.outputs.cpu }} + containerMemory: ${{ steps.provider.outputs.memory }} + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +## Fallback on Runner Availability + +If your self-hosted runner is offline or busy, fall back to a cloud provider. Use a short initial +job to check runner health, then pick the provider accordingly. + +```yaml +name: Fallback Routing + +on: + push: + branches: [main] + +jobs: + check-runner: + name: Check self-hosted availability + runs-on: ubuntu-latest + outputs: + provider: ${{ steps.pick.outputs.provider }} + steps: + - name: Check if self-hosted runner is available + id: pick + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Query runner status via the GitHub API + RUNNERS=$(gh api repos/${{ github.repository }}/actions/runners --jq '[.runners[] | select(.status == "online")] | length') + if [[ "$RUNNERS" -gt 0 ]]; then + echo "provider=local-docker" >> "$GITHUB_OUTPUT" + echo "Self-hosted runner available — using local-docker" + else + echo "provider=aws" >> "$GITHUB_OUTPUT" + echo "No runners online — falling back to AWS" + fi + + build: + name: Build + needs: check-runner + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + - uses: game-ci/unity-builder@v4 + with: + providerStrategy: ${{ needs.check-runner.outputs.provider }} + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +## Async Mode for Load Balancing + +The [`asyncOrchestrator`](../api-reference#github-integration) parameter works well with load +balancing. When enabled, the GitHub Action dispatches the build and returns immediately without +waiting for it to finish. This means your routing job completes in seconds regardless of which +provider handles the actual build. + +``` + Router Job (seconds) Provider A Provider B + ┌───────────────────┐ ┌──────────────┐ ┌──────────────┐ + │ 1. Check runners │ │ │ │ │ + │ 2. Pick provider │ │ │ │ │ + │ 3. Dispatch build ├──────►│ 3a. Building │ OR ─►│ 3b. Building │ + │ 4. Return (done) │ │ ... │ │ ... │ + └───────────────────┘ │ ... │ │ ... │ + Completes instantly │ 5. Complete │ │ 5. Complete │ + └──────┬───────┘ └──────┬───────┘ + │ │ + ┌──────▼───────────────────────▼──────┐ + │ GitHub Check updated │ + │ (monitor from PR page) │ + └─────────────────────────────────────┘ +``` + +Benefits of combining async mode with load balancing: + +- **No wasted runner minutes** — The router job finishes in seconds. You don't pay for a + GitHub-hosted runner to sit idle while the build runs in the cloud. +- **Parallel dispatch** — Launch multiple builds to different providers simultaneously. Each + dispatches and returns immediately. +- **Monitor everything from one place** — Enable `githubCheck: true` and all builds report status + back to the pull request Checks tab, regardless of which provider ran them. + +```yaml +jobs: + build: + name: Build ${{ matrix.targetPlatform }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - targetPlatform: StandaloneLinux64 + provider: aws + - targetPlatform: StandaloneWindows64 + provider: k8s + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + - uses: game-ci/unity-builder@v4 + with: + providerStrategy: ${{ matrix.provider }} + targetPlatform: ${{ matrix.targetPlatform }} + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} + asyncOrchestrator: true + githubCheck: true +``` + +Each matrix job dispatches its build and exits immediately. The builds continue running on their +respective providers and report progress via GitHub Checks. + +## Weighted Distribution + +For teams with both cloud and self-hosted capacity, distribute builds based on a ratio. This example +sends 70% of builds to AWS and 30% to a local runner using a simple hash. + +```yaml +- name: Weighted provider selection + id: provider + run: | + # Hash the run ID to get a stable pseudo-random number + HASH=$(echo "${{ github.run_id }}" | md5sum | cut -c1-8) + DECIMAL=$((16#$HASH % 100)) + if [[ $DECIMAL -lt 70 ]]; then + echo "strategy=aws" >> "$GITHUB_OUTPUT" + else + echo "strategy=local-docker" >> "$GITHUB_OUTPUT" + fi +``` + +## Tips + +- **Start simple** — A platform-based matrix with different providers per entry is the easiest + starting point. Add dynamic routing only when you need it. +- **Use async for long builds** — Combining `asyncOrchestrator: true` with `githubCheck: true` keeps + your routing job fast and gives you build status on the PR page. +- **Cache keys are provider-independent** — The [`cacheKey`](../api-reference#caching) parameter + works the same across all providers, so builds routed to different providers can still share + caches if they use the same storage backend. +- **Test fallback logic** — Temporarily disable your self-hosted runner to verify that your fallback + routing works before you need it in production. +- **Custom providers** — The same routing patterns work with + [custom providers](../providers/custom-providers). Set `providerStrategy` to a GitHub repo or NPM + package and Orchestrator loads it dynamically. From cf9039be18f8fd699634b826b061b5a94e5ed398 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 06:54:45 +0000 Subject: [PATCH 17/20] Add Storage, Architecture pages and Build Caching section - Storage page: documents project files, build output, caches, S3 and rclone backends, LZ4 compression, workspace locking, large packages, and container file system layout - Architecture page: describes build lifecycle, core components, provider system, workflow composition, hook system, configuration resolution, remote client, CLI modes, and source code map - Caching page: add Build Caching section explaining automatic build output caching based on cache key Co-Authored-By: Claude Opus 4.6 --- .../07-advanced-topics/01-caching.mdx | 22 ++ .../07-advanced-topics/08-storage.mdx | 215 ++++++++++++ .../07-advanced-topics/09-architecture.mdx | 308 ++++++++++++++++++ 3 files changed, 545 insertions(+) create mode 100644 docs/03-github-orchestrator/07-advanced-topics/08-storage.mdx create mode 100644 docs/03-github-orchestrator/07-advanced-topics/09-architecture.mdx diff --git a/docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx b/docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx index be546e84..cc458ebb 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/01-caching.mdx @@ -25,6 +25,28 @@ requires re-importing unchanged assets. - ✅ Best for smaller projects - ⚠️ Slower rebuilds for large asset libraries +## Build Caching + +Orchestrator automatically caches build output alongside the Library cache. After each build, the +compiled output folder is archived and stored using the same cache key (branch name by default). On +the next build with the same cache key, the previous build output is available at +`/data/cache/{cacheKey}/build/`. + +This happens automatically — no configuration required. The cache key controls which builds share +output: + +```yaml +# Builds on the same branch share cached output (default behavior) +cacheKey: ${{ github.ref_name }} + +# Or share across branches by using a fixed key +cacheKey: shared-cache +``` + +Build caching uses the same compression and storage provider as Library caching. Archives are stored +as `build-{buildGuid}.tar.lz4` (or `.tar` if compression is disabled). See [Storage](storage) for +details on compression and storage backends. + ## Retained Workspace Caches the **entire project folder** between builds. Provides the fastest rebuilds but consumes more diff --git a/docs/03-github-orchestrator/07-advanced-topics/08-storage.mdx b/docs/03-github-orchestrator/07-advanced-topics/08-storage.mdx new file mode 100644 index 00000000..900caa39 --- /dev/null +++ b/docs/03-github-orchestrator/07-advanced-topics/08-storage.mdx @@ -0,0 +1,215 @@ +# Storage + +Orchestrator manages three categories of files during a build: **project files** (your Unity project +and Git LFS assets), **build output** (the compiled game), and **caches** (Unity Library folder and +LFS files). This page explains how each category flows through the system and how to configure the +storage backend. + +``` + CI / Local Machine Build Container + ┌──────────────────┐ ┌──────────────────────────────────┐ + │ Git repository │──── clone ────►│ /data/{buildGuid}/repo/ │ + │ + LFS assets │ │ ├── Assets/ │ + │ │ │ ├── Library/ (cached) │ + └──────────────────┘ │ └── .git/lfs/ (cached) │ + │ │ + │ /data/cache/{cacheKey}/ │ + │ ├── Library/ (tar.lz4) │ + │ └── build/ (tar.lz4) │ + └────────────┬─────────────────────┘ + │ + ┌────────────▼─────────────────────┐ + │ Cloud Storage (S3 / rclone) │ + │ ├── Library cache archives │ + │ ├── Build artifacts │ + │ └── Workspace locks │ + └──────────────────────────────────┘ +``` + +## File Categories + +### Project Files + +Your Unity project is cloned into the build container at `/data/{buildGuid}/repo/`. Orchestrator +handles Git and LFS automatically: + +1. **Shallow clone** — The repository is cloned with `--depth` controlled by the + [`cloneDepth`](../api-reference#git-synchronization) parameter (default: 50). +2. **LFS pull** — Git LFS is installed and configured inside the container. LFS files are pulled + after the clone completes. +3. **LFS hashing** — Orchestrator generates `.lfs-assets-guid` and `.lfs-assets-guid-sum` files to + track LFS content for cache invalidation. + +For retained workspaces, the project folder persists between builds at +`/data/{lockedWorkspace}/repo/` instead of being cloned fresh each time. See +[Retained Workspaces](retained-workspace) for details. + +### Build Output + +After Unity finishes building, the compiled output lives at `/data/{buildGuid}/repo/{buildPath}/`. +Orchestrator archives this folder as `build-{buildGuid}.tar` (or `.tar.lz4` with compression +enabled). + +To export build artifacts out of the container, use [container hooks](hooks/container-hooks). The +most common approach is the built-in S3 or rclone upload hooks: + +```yaml +# Upload build artifacts to S3 +containerHookFiles: aws-s3-upload-build + +# Or upload via rclone to any backend +containerHookFiles: rclone-upload-build +``` + +See [Built-In Hooks](hooks/built-in-hooks) for all available hooks. + +### Caches + +Orchestrator caches two things between builds to speed up subsequent runs: + +| Cache | Contents | Path in container | +| ------------- | ------------------------------------------- | --------------------------------- | +| Library cache | Unity's `Library/` folder (imported assets) | `/data/cache/{cacheKey}/Library/` | +| Build cache | Previous build output | `/data/cache/{cacheKey}/build/` | + +Caches are scoped by the [`cacheKey`](../api-reference#caching) parameter, which defaults to the +branch name. Builds on the same branch share a cache. + +After a build completes, Orchestrator runs `remote-cli-post-build` which: + +1. Archives the Library folder as `lib-{buildGuid}.tar` (or `.tar.lz4`). +2. Archives the build output as `build-{buildGuid}.tar` (or `.tar.lz4`). +3. Pushes both archives to cloud storage via the configured storage provider. + +Before the next build, `remote-cli-pre-build` pulls these archives and extracts them, so Unity can +skip re-importing unchanged assets. + +## Storage Providers + +Orchestrator supports two storage backends for caches, artifacts, and workspace locks. + +``` + storageProvider: "s3" storageProvider: "rclone" + ┌────────────────────┐ ┌────────────────────┐ + │ AWS S3 │ │ Rclone │ + │ │ │ │ + │ - Default backend │ │ - 70+ backends │ + │ - Works with │ │ - Google Cloud │ + │ LocalStack │ │ - Azure Blob │ + │ - Built-in lock │ │ - Backblaze B2 │ + │ support │ │ - SFTP, FTP │ + │ │ │ - Any rclone │ + │ │ │ remote │ + └────────────────────┘ └────────────────────┘ +``` + +### S3 (Default) + +S3 is the default storage backend. It works with AWS S3 and LocalStack (for local testing). + +No extra configuration is needed when using the `aws` provider — the S3 bucket is created +automatically as part of the CloudFormation base stack. For other providers, ensure AWS credentials +and region are set in the environment. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + # storageProvider defaults to "s3" + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +### Rclone + +[Rclone](https://rclone.org) is a command-line tool that supports 70+ cloud storage backends. Use it +when you want to store caches and artifacts somewhere other than S3. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: k8s + storageProvider: rclone + rcloneRemote: 'myremote:bucket/path' + targetPlatform: StandaloneLinux64 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +Rclone hooks run in the `rclone/rclone` Docker image. Configure your rclone remote beforehand (see +[rclone docs](https://rclone.org/docs/)). + +**Supported backends include:** Google Cloud Storage, Azure Blob, Backblaze B2, DigitalOcean Spaces, +Wasabi, MinIO, SFTP, FTP, and [many more](https://rclone.org/overview/). + +## Compression + +Orchestrator uses **LZ4 compression** by default for all cache and build archives. LZ4 is optimized +for speed over compression ratio, which is ideal for large Unity Library folders where fast +decompression matters more than file size. + +| `useCompressionStrategy` | Archive format | Description | +| ------------------------ | -------------- | ------------------------------------------------ | +| `true` (default) | `.tar.lz4` | LZ4 compressed. Faster extract, ~30-50% smaller. | +| `false` | `.tar` | Uncompressed. Slightly faster to create. | + +```yaml +# Disable compression (not recommended for most projects) +useCompressionStrategy: false +``` + +## Workspace Locking + +When using [retained workspaces](retained-workspace), Orchestrator uses distributed locking to +ensure only one build occupies a workspace at a time. Locks are stored in the configured storage +provider: + +- **S3**: Lock files at `s3://{awsStackName}/locks/{workspaceName}/{buildGuid}` +- **Rclone**: Lock files at `{rcloneRemote}/locks/{workspaceName}/{buildGuid}` + +Locking is fully automatic. When a build starts, it acquires a lock. When it finishes (or fails), +the lock is released. If all workspaces are locked, the build falls back to standard caching. + +## Large Packages + +The [`useLargePackages`](../api-reference#build-options) parameter optimizes storage for Unity +packages containing "LargePackage" in `manifest.json`. When enabled, these packages are redirected +to a shared folder so multiple builds with the same cache key share one copy instead of duplicating +data. + +```yaml +useLargePackages: true +``` + +## Container File System Layout + +Inside the build container, Orchestrator uses the `/data/` volume mount as the root for all project +and cache data. + +``` +/data/ +├── {buildGuid}/ # Unique job folder (or {lockedWorkspace}/) +│ ├── repo/ # Cloned repository +│ │ ├── Assets/ +│ │ ├── Library/ # Unity Library (restored from cache) +│ │ ├── .git/ +│ │ │ └── lfs/ # Git LFS objects +│ │ └── {buildPath}/ # Build output +│ └── builder/ # Cloned game-ci/unity-builder +│ └── dist/ +│ └── index.js +└── cache/ + └── {cacheKey}/ # Scoped by branch name + ├── Library/ + │ └── lib-{guid}.tar.lz4 # Library cache archive + └── build/ + └── build-{guid}.tar.lz4 # Build output archive +``` + +## Storage Parameters + +For the full list of storage-related parameters, see the API Reference: + +- [Storage](../api-reference#storage) — `storageProvider`, `rcloneRemote` +- [Caching](../api-reference#caching) — `cacheKey`, `maxRetainedWorkspaces` +- [Build Options](../api-reference#build-options) — `useCompressionStrategy`, `useLargePackages` +- [AWS](../api-reference#aws) — `awsStackName`, `awsS3Endpoint` diff --git a/docs/03-github-orchestrator/07-advanced-topics/09-architecture.mdx b/docs/03-github-orchestrator/07-advanced-topics/09-architecture.mdx new file mode 100644 index 00000000..fc838dc1 --- /dev/null +++ b/docs/03-github-orchestrator/07-advanced-topics/09-architecture.mdx @@ -0,0 +1,308 @@ +# Architecture + +This page describes the internal architecture of Orchestrator — how the components fit together, how +a build flows through the system, and where to look in the source code. + +``` + ┌───────────────────────────┐ + │ Entry Point │ + │ GitHub Action / CLI │ + └─────────────┬─────────────┘ + │ + ┌─────────────▼─────────────┐ + │ Orchestrator │ + │ (orchestrator.ts) │ + │ │ + │ - setup() │ + │ - run() │ + │ - Provider selection │ + └──┬──────────┬──────────┬──┘ + │ │ │ + ┌──────────────────▼┐ ┌──────▼───────┐ ┌▼──────────────────┐ + │ Provider │ │ Workflow │ │ Services │ + │ (pluggable) │ │ Composition │ │ │ + │ │ │ Root │ │ - Logger │ + │ - aws │ │ │ │ - Hooks │ + │ - k8s │ │ - Standard │ │ - Caching │ + │ - local-docker │ │ - Async │ │ - Locking │ + │ - local │ │ - Custom Job │ │ - LFS │ + │ - custom plugin │ │ │ │ - GitHub Checks │ + └────────────────────┘ └──────────────┘ └────────────────────┘ +``` + +## Build Lifecycle + +A standard Orchestrator build follows these steps: + +``` + 1. Initialize 2. Setup Provider 3. Acquire Workspace + ┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ + │ Parse inputs │ │ Create cloud │ │ Lock retained │ + │ Select provider │──►│ resources │──►│ workspace │ + │ Generate GUID │ │ (stacks, PVCs) │ │ (if enabled) │ + │ Create GH check │ │ │ │ │ + └──────────────────┘ └──────────────────┘ └──────────────────┘ + │ + 6. Cleanup 5. Post-Build 4. Build + ┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ + │ Release workspace│ │ Push Library │ │ Clone repo + LFS │ + │ Delete cloud │◄──│ and build cache │◄──│ Restore cache │ + │ resources │ │ Run post hooks │ │ Run Unity build │ + │ Update GH check │ │ │ │ Run pre/post │ + └──────────────────┘ └──────────────────┘ │ container hooks │ + └──────────────────┘ +``` + +### Step-by-Step + +1. **Initialize** — `Orchestrator.setup()` parses inputs from GitHub Action `with`, CLI flags, or + environment variables. It selects a provider, generates a unique build GUID, and optionally + creates a GitHub Check. + +2. **Setup Provider** — `Provider.setupWorkflow()` provisions cloud resources. For AWS this means + creating a CloudFormation base stack (ECS cluster, S3 bucket, Kinesis stream). For Kubernetes it + creates a service account. Local providers skip this step. + +3. **Acquire Workspace** — If `maxRetainedWorkspaces` is set, `SharedWorkspaceLocking` acquires a + distributed lock on a workspace via S3 or rclone. If all workspaces are locked, the build falls + back to standard caching. + +4. **Build** — The workflow composition root selects a workflow (standard, async, or custom job). + The standard workflow clones your repo, restores caches, runs the Unity build, and executes + pre/post container hooks. + +5. **Post-Build** — `remote-cli-post-build` archives the Unity Library folder and build output, + pushes them to cloud storage, and runs any post-build hooks. + +6. **Cleanup** — The workspace lock is released, cloud resources are torn down + (`Provider.cleanupWorkflow()`), and the GitHub Check is updated with the final result. + +## Core Components + +### Orchestrator (`orchestrator.ts`) + +The static `Orchestrator` class is the central coordinator. It holds: + +- `Provider` — the selected provider implementation +- `buildParameters` — all resolved configuration +- `buildGuid` — unique identifier for this build +- `lockedWorkspace` — retained workspace name (if any) + +`Orchestrator.run()` is the main entry point that drives the full lifecycle. Provider selection +happens in `setupSelectedBuildPlatform()`, which handles LocalStack detection, `AWS_FORCE_PROVIDER` +overrides, and fallback to the plugin loader for custom providers. + +### Provider System + +Providers implement the `ProviderInterface`: + +```typescript +interface ProviderInterface { + setupWorkflow(buildGuid, buildParameters, branchName, secrets): Promise; + runTaskInWorkflow( + buildGuid, + image, + commands, + mountdir, + workingdir, + env, + secrets, + ): Promise; + cleanupWorkflow(buildParameters, branchName, secrets): Promise; + garbageCollect(filter, previewOnly, olderThan, fullCache, baseDependencies): Promise; + listResources(): Promise; + listWorkflow(): Promise; + watchWorkflow(): Promise; +} +``` + +Each provider handles `runTaskInWorkflow()` differently: + +| Provider | How it runs the build | +| -------------- | -------------------------------------------------------------- | +| `aws` | Creates a CloudFormation job stack, starts an ECS Fargate task | +| `k8s` | Creates a PVC, Kubernetes Secret, and Job; streams pod logs | +| `local-docker` | Runs a Docker container with volume mounts | +| `local` | Executes shell commands directly on the host | + +#### Custom Provider Loading + +When `providerStrategy` doesn't match a built-in name, the provider loader: + +1. Parses the source (GitHub URL, NPM package, or local path) via `ProviderUrlParser` +2. Clones or installs the module via `ProviderGitManager` +3. Validates that all 7 interface methods exist +4. Falls back to the local provider if loading fails + +See [Custom Providers](../providers/custom-providers) for the user-facing guide. + +### Workflow System + +The `WorkflowCompositionRoot` selects which workflow to run: + +``` + asyncOrchestrator: true? + │ + ┌────▼────┐ + │ Yes │──► AsyncWorkflow + └────┬────┘ Dispatches build to cloud container + │ and returns immediately + ┌────▼────┐ + │ No │ + └────┬────┘ + │ + customJob set? + │ + ┌────▼────┐ + │ Yes │──► CustomWorkflow + └────┬────┘ Parses YAML job definition + │ and runs container steps + ┌────▼────┐ + │ No │──► BuildAutomationWorkflow + └─────────┘ Standard build pipeline +``` + +**BuildAutomationWorkflow** generates a shell script that runs inside the container. The script: + +1. Installs toolchain (Node.js, npm, yarn, git-lfs) for remote providers +2. Clones game-ci/unity-builder into the container +3. Runs `remote-cli-pre-build` (restores caches, clones the target repo) +4. Executes the Unity build via the standard Game CI entrypoint +5. Runs `remote-cli-post-build` (pushes caches) +6. Writes log markers for collection + +**AsyncWorkflow** runs the entire build inside a cloud container. It installs the AWS CLI, clones +both the builder and target repos, and executes `index.js -m async-workflow`. The calling GitHub +Action returns immediately. Progress is reported via +[GitHub Checks](../providers/github-integration). + +### Hook System + +Orchestrator has two hook mechanisms: + +**Command Hooks** — Shell commands injected before or after the setup and build steps. Defined via +the `customCommandHooks` YAML parameter or as files in `.game-ci/command-hooks/`. + +**Container Hooks** — Separate Docker containers that run before or after the build. Defined via +`containerHookFiles` (built-in names like `aws-s3-upload-build`) or `preBuildSteps` / +`postBuildSteps` YAML. Each hook specifies an image, commands, and optional secrets. + +See [Hooks](hooks/container-hooks) for the full guide. + +### Configuration Resolution + +Orchestrator reads configuration from multiple sources with this priority: + +``` + Highest Priority + ┌──────────────────────┐ + │ GitHub Action inputs │ with: providerStrategy: aws + ├──────────────────────┤ + │ CLI flags │ --providerStrategy aws + ├──────────────────────┤ + │ Query overrides │ Pull Secrets from external sources + ├──────────────────────┤ + │ Environment variables │ PROVIDER_STRATEGY=aws + └──────────────────────┘ + Lowest Priority +``` + +The `OrchestratorOptions` class handles this resolution. Environment variables accept both +`camelCase` and `UPPER_SNAKE_CASE` formats. + +### Remote Client + +The remote client runs **inside** the build container (not on the CI runner). It provides two CLI +modes: + +- **`remote-cli-pre-build`** — Called before the Unity build. Handles git clone, LFS pull, cache + restoration, retained workspace setup, large package optimization, and custom hook execution. +- **`remote-cli-post-build`** — Called after the Unity build. Pushes the Library and build caches to + cloud storage. + +### GitHub Integration + +The `GitHub` class manages GitHub Checks and async workflow dispatch: + +- **`createGitHubCheck()`** — Creates a check run on the commit via the GitHub API. +- **`updateGitHubCheck()`** — Updates check status. In async environments, updates are routed + through the `Async Checks API` workflow (since containers can't call the Checks API directly). +- **`runUpdateAsyncChecksWorkflow()`** — Triggers a GitHub Actions workflow that updates the check + run on behalf of the container. + +### Caching and Storage + +Caching is split between the remote client (push/pull logic) and the storage provider (S3 or +rclone): + +- **Standard caching** — Archives the `Library/` folder and LFS files as `.tar.lz4` archives. +- **Retained workspaces** — Keeps the entire project folder. Uses distributed locking via S3 or + rclone to prevent concurrent access. + +See [Storage](storage) for the full breakdown of file categories, compression, and storage backends. + +## CLI Modes + +The CLI system uses a decorator-based registry (`@CliFunction`). Each mode maps to a static method: + +| Mode | Description | +| ----------------------- | ---------------------------------------------------- | +| `cli-build` | Full build workflow (default) | +| `async-workflow` | Async build execution (called from within container) | +| `remote-cli-pre-build` | Pre-build setup (runs inside container) | +| `remote-cli-post-build` | Post-build cache push (runs inside container) | +| `remote-cli-log-stream` | Pipe and capture build logs | +| `checks-update` | Update GitHub Checks from async container | +| `cache-push` | Push a directory to cache storage | +| `cache-pull` | Pull a directory from cache storage | +| `garbage-collect` | Clean up old cloud resources | +| `list-resources` | List active cloud resources | +| `list-workflow` | List running workflows | +| `watch` | Follow logs of a running workflow | +| `hash` | Hash folder contents recursively | +| `print-input` | Print all resolved parameters | + +## Source Code Map + +``` +src/model/orchestrator/ +├── orchestrator.ts # Main coordinator +├── options/ +│ ├── orchestrator-options.ts # Configuration resolution +│ └── orchestrator-folders.ts # Path management (/data/...) +├── workflows/ +│ ├── workflow-composition-root.ts # Workflow selection +│ ├── build-automation-workflow.ts # Standard build pipeline +│ ├── async-workflow.ts # Async dispatch +│ └── custom-workflow.ts # Custom job execution +├── providers/ +│ ├── provider-interface.ts # 7-method contract +│ ├── provider-loader.ts # Dynamic plugin loading +│ ├── provider-url-parser.ts # GitHub/NPM/local parsing +│ ├── provider-git-manager.ts # Clone and cache repos +│ ├── aws/ # AWS Fargate provider +│ ├── k8s/ # Kubernetes provider +│ ├── docker/ # Local Docker provider +│ └── local/ # Direct execution provider +├── services/ +│ ├── core/ +│ │ ├── orchestrator-logger.ts +│ │ ├── orchestrator-system.ts # Shell command execution +│ │ ├── shared-workspace-locking.ts +│ │ └── follow-log-stream-service.ts +│ ├── hooks/ +│ │ ├── command-hook-service.ts +│ │ └── container-hook-service.ts +│ └── cache/ +│ └── local-cache-service.ts +├── remote-client/ +│ ├── index.ts # Pre/post build logic +│ └── caching.ts # Cache push/pull with LZ4 +└── tests/ + +src/model/cli/ +├── cli.ts # CLI entry point +└── cli-functions-repository.ts # @CliFunction registry + +src/model/github.ts # GitHub Checks + async dispatch +``` From a99718604103b45865a550ea5fffc30abb3ec099 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 07:05:26 +0000 Subject: [PATCH 18/20] Fix introduction diagram to list all supported CI platforms The box previously said "GitHub Actions" which contradicted the "Your Machine / CI" header. Now lists GitHub Actions, GitLab CI, CLI, etc. to reflect that Orchestrator works from any entry point. Co-Authored-By: Claude Opus 4.6 --- docs/03-github-orchestrator/01-introduction.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/03-github-orchestrator/01-introduction.mdx b/docs/03-github-orchestrator/01-introduction.mdx index 1060b6b3..0f222435 100644 --- a/docs/03-github-orchestrator/01-introduction.mdx +++ b/docs/03-github-orchestrator/01-introduction.mdx @@ -9,10 +9,10 @@ be built, and streams results back. ``` Your Machine / CI Cloud Provider ┌──────────────┐ git push ┌─────────────────┐ - │ │ ─────────────────►│ AWS Fargate │ - │ GitHub │ │ Kubernetes │ - │ Actions │ ◄─────────────────│ Local Docker │ - │ │ build artifacts │ │ + │ GitHub │ ─────────────────►│ AWS Fargate │ + │ Actions, │ │ Kubernetes │ + │ GitLab CI, │ ◄─────────────────│ Local Docker │ + │ CLI, etc. │ build artifacts │ │ └──────────────┘ └─────────────────┘ │ │ │ Orchestrator handles: │ From 7b370f197214e1825cb14c1ae99988017c6d3b56 Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 07:20:40 +0000 Subject: [PATCH 19/20] Fix ASCII diagram alignment in load balancing page Co-Authored-By: Claude Opus 4.6 --- .../07-advanced-topics/07-load-balancing.mdx | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/03-github-orchestrator/07-advanced-topics/07-load-balancing.mdx b/docs/03-github-orchestrator/07-advanced-topics/07-load-balancing.mdx index eb527e54..dcd45b12 100644 --- a/docs/03-github-orchestrator/07-advanced-topics/07-load-balancing.mdx +++ b/docs/03-github-orchestrator/07-advanced-topics/07-load-balancing.mdx @@ -5,23 +5,23 @@ builds across them. This lets you distribute builds based on platform, project s availability, or any custom logic. ``` - ┌──────────────────────┐ - │ Incoming Build │ - │ Request │ - └──────────┬─────────────┘ - │ - ┌──────────▼─────────────┐ - │ Router Logic │ - │ (GitHub Actions / │ - │ script / matrix) │ - └──┬───────┬──────────┬───┘ - │ │ │ - ┌──────────────▼┐ ┌───▼────────┐ ┌▼──────────────┐ - │ aws │ │ k8s │ │ local-docker │ - │ │ │ │ │ │ - │ Fargate │ │ Cluster │ │ Self-hosted │ - │ (scalable) │ │ (flexible) │ │ (no cost) │ - └───────────────┘ └────────────┘ └───────────────┘ + ┌─────────────────────┐ + │ Incoming Build │ + │ Request │ + └─────────┬───────────┘ + │ + ┌─────────▼───────────┐ + │ Router Logic │ + │ (GitHub Actions / │ + │ script / matrix) │ + └──┬──────┬────────┬──┘ + │ │ │ + ┌───────────▼──┐ ┌─▼──────────┐ ┌▼─────────────┐ + │ aws │ │ k8s │ │ local-docker │ + │ │ │ │ │ │ + │ Fargate │ │ Cluster │ │ Self-hosted │ + │ (scalable) │ │ (flexible) │ │ (no cost) │ + └──────────────┘ └────────────┘ └──────────────┘ ``` ## Why Load Balance? From 93c86dac446bc0ee81dfd2c5a479836deb4e131e Mon Sep 17 00:00:00 2001 From: frostebite Date: Thu, 5 Mar 2026 08:48:27 +0000 Subject: [PATCH 20/20] docs: add Orchestrator Jobs page and Custom LFS Agents page New Jobs page explains the build lifecycle, job types (build, test, custom editor method, custom job, async), pre/post build phases, and execution by provider. Gives users a conceptual overview before diving into advanced topics. New LFS Agents page documents elastic-git-storage built-in support with auto-install, version pinning, multiple storage backends, and custom agent configuration. Renamed api-reference from 04 to 05 to accommodate the new Jobs page. Co-Authored-By: Claude Opus 4.6 --- docs/03-github-orchestrator/04-jobs.mdx | 189 ++++++++++++++++++ ...api-reference.mdx => 05-api-reference.mdx} | 0 .../07-advanced-topics/10-lfs-agents.mdx | 92 +++++++++ 3 files changed, 281 insertions(+) create mode 100644 docs/03-github-orchestrator/04-jobs.mdx rename docs/03-github-orchestrator/{04-api-reference.mdx => 05-api-reference.mdx} (100%) create mode 100644 docs/03-github-orchestrator/07-advanced-topics/10-lfs-agents.mdx diff --git a/docs/03-github-orchestrator/04-jobs.mdx b/docs/03-github-orchestrator/04-jobs.mdx new file mode 100644 index 00000000..5318cbd0 --- /dev/null +++ b/docs/03-github-orchestrator/04-jobs.mdx @@ -0,0 +1,189 @@ +# Orchestrator Jobs + +Orchestrator executes work as **jobs** — containerized or local tasks that run on your chosen +provider. Understanding job types and their flow is key to customizing your build pipeline. + +## Job Flow + +Every Orchestrator run follows the same lifecycle, regardless of provider: + +``` + ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ + │ Setup │────►│ Pre-Build │────►│ Build │────►│ Post-Build │ + │ Provider │ │ Jobs │ │ Job │ │ Jobs │ + └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ + │ │ + │ ┌─────────────┐ │ + └───────────────────►│ Cleanup │◄────────────────────────┘ + └─────────────┘ +``` + +1. **Setup** — Provision cloud resources (stacks, volumes, secrets). Skipped for local providers. +2. **Pre-build jobs** — Clone the repository, pull LFS, restore caches, run pre-build hooks. +3. **Build job** — Execute the Unity build (or custom editor method). +4. **Post-build jobs** — Push caches, upload artifacts, run post-build hooks. +5. **Cleanup** — Release locks, tear down cloud resources, update GitHub Checks. + +## Job Types + +### Build Job + +The standard job — runs the Unity Editor to produce a build artifact. This is what most users +care about. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + targetPlatform: StandaloneLinux64 + providerStrategy: aws + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +The build job: +- Installs the toolchain (Node.js, git-lfs) inside the container +- Clones `game-ci/unity-builder` into the container +- Runs `remote-cli-pre-build` to set up the workspace +- Executes the Unity build via the Game CI entrypoint +- Runs `remote-cli-post-build` to push caches and artifacts + +### Test Jobs + +Run Unity tests without producing a build. Use a custom `buildMethod` that runs tests and exits: + +```yaml +- uses: game-ci/unity-builder@v4 + with: + targetPlatform: StandaloneLinux64 + buildMethod: MyNamespace.TestRunner.RunEditModeTests + manualExit: true + providerStrategy: aws + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +With `manualExit: true`, Unity doesn't quit automatically — your build method should call +`EditorApplication.Exit(0)` after tests complete. This gives you full control over the test +lifecycle. + +### Custom Editor Method Jobs + +Run any static C# method in the Unity Editor. Useful for: +- Asset processing or validation +- Addressables builds +- Custom pipeline steps +- Code generation + +```yaml +- uses: game-ci/unity-builder@v4 + with: + targetPlatform: StandaloneLinux64 + buildMethod: MyNamespace.Pipeline.ProcessAssets + manualExit: true + providerStrategy: aws + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +The `buildMethod` must be a fully qualified static method: `Namespace.Class.Method`. + +### Custom Jobs + +Replace the entire build workflow with your own container steps. Useful for non-Unity workloads +or fully custom pipelines that still benefit from Orchestrator's cloud infrastructure. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + providerStrategy: aws + customJob: | + - name: my-custom-step + image: ubuntu:22.04 + commands: | + echo "Running custom workload" + ./my-script.sh + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +See [Custom Job](advanced-topics/custom-job) for the full reference. + +### Async Jobs + +For long-running builds, Orchestrator can dispatch the job and return immediately. The build +continues in the cloud. Progress is reported via GitHub Checks. + +```yaml +- uses: game-ci/unity-builder@v4 + with: + targetPlatform: StandaloneLinux64 + providerStrategy: aws + asyncOrchestrator: true + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} +``` + +This is useful when builds exceed GitHub Actions' job time limits, or when you want to +free up your CI runner immediately. + +## Pre-Build and Post-Build + +Orchestrator runs additional steps before and after the main build job. + +### Pre-Build Steps + +The `remote-cli-pre-build` phase handles: +- Git clone of the target repository +- Git LFS pull +- Cache restoration (Library folder, LFS objects) +- Retained workspace setup +- Submodule initialization (if using [submodule profiles](advanced-topics/submodule-profiles)) +- Custom LFS agent configuration (e.g., [elastic-git-storage](advanced-topics/lfs-agents)) + +You can inject additional pre-build steps: + +```yaml +- uses: game-ci/unity-builder@v4 + with: + preBuildSteps: | + - name: install-dependencies + image: node:18 + commands: npm install +``` + +### Post-Build Steps + +The `remote-cli-post-build` phase handles: +- Library folder cache push +- Build artifact upload +- LFS cache push + +You can inject additional post-build steps: + +```yaml +- uses: game-ci/unity-builder@v4 + with: + postBuildSteps: | + - name: upload-to-steam + image: steamcmd + commands: ./upload.sh +``` + +### Hooks + +For more granular control, use [container hooks](advanced-topics/hooks/container-hooks) or +[command hooks](advanced-topics/hooks/command-hooks) to inject steps at specific points in the +build lifecycle. + +## Job Execution by Provider + +Each provider runs jobs differently: + +| Provider | How jobs execute | +|----------|----------------| +| **AWS Fargate** | ECS Fargate task with CloudFormation stack. Logs streamed via Kinesis. | +| **Kubernetes** | Kubernetes Job with PVC for workspace. Logs streamed from pod. | +| **Local Docker** | Docker container with volume mounts. Logs piped to stdout. | +| **Local** | Direct shell execution on the host. No container isolation. | +| **CLI Provider** | Delegated to external executable via JSON protocol. | + +## Next Steps + +- [Custom Job](advanced-topics/custom-job) — Full reference for custom job definitions +- [Hooks](advanced-topics/hooks/container-hooks) — Inject steps at specific lifecycle points +- [Architecture](advanced-topics/architecture) — Deep dive into internal components diff --git a/docs/03-github-orchestrator/04-api-reference.mdx b/docs/03-github-orchestrator/05-api-reference.mdx similarity index 100% rename from docs/03-github-orchestrator/04-api-reference.mdx rename to docs/03-github-orchestrator/05-api-reference.mdx diff --git a/docs/03-github-orchestrator/07-advanced-topics/10-lfs-agents.mdx b/docs/03-github-orchestrator/07-advanced-topics/10-lfs-agents.mdx new file mode 100644 index 00000000..c28adaa9 --- /dev/null +++ b/docs/03-github-orchestrator/07-advanced-topics/10-lfs-agents.mdx @@ -0,0 +1,92 @@ +# Custom LFS Agents + +Orchestrator supports custom Git LFS transfer agents — external executables that handle +LFS upload and download instead of the default HTTPS transport. + +## elastic-git-storage (Built-in) + +[elastic-git-storage](https://github.com/frostebite/elastic-git-storage) is a custom Git LFS +transfer agent with first-class support in Orchestrator. It supports multiple storage backends: +local filesystem, WebDAV, and rclone remotes. + +When you set `lfsTransferAgent: elastic-git-storage`, Orchestrator will: + +1. Search PATH and known locations for an existing installation +2. If not found, download the correct platform binary from GitHub releases +3. Configure it as the standalone LFS transfer agent via `git config` + +### Basic Usage + +```yaml +- uses: game-ci/unity-builder@v4 + with: + targetPlatform: StandaloneLinux64 + lfsTransferAgent: elastic-git-storage + lfsStoragePaths: '/mnt/lfs-cache' +``` + +### Version Pinning + +Append `@version` to pin a specific release: + +```yaml +lfsTransferAgent: elastic-git-storage@v1.0.0 +``` + +Without a version suffix, the latest release is downloaded. + +### Multiple Storage Backends + +`lfsStoragePaths` accepts semicolon-separated paths. The agent tries each in order: + +```yaml +lfsStoragePaths: '/mnt/fast-ssd;webdav://lfs.example.com/storage;rclone://remote:lfs-bucket' +``` + +### Agent Arguments + +Pass additional flags via `lfsTransferAgentArgs`: + +```yaml +lfsTransferAgent: elastic-git-storage +lfsTransferAgentArgs: '--verbose --concurrency 4' +lfsStoragePaths: '/mnt/lfs-cache' +``` + +## Custom Agents + +Any Git LFS custom transfer agent can be used. Provide the path to the executable: + +```yaml +lfsTransferAgent: /usr/local/bin/lfs-folderstore +lfsTransferAgentArgs: '-dir /mnt/lfs-store' +``` + +Orchestrator configures the agent via: + +``` +git config lfs.customtransfer..path +git config lfs.customtransfer..args +git config lfs.standalonetransferagent +``` + +The agent name is derived from the executable filename (without extension). + +## Storage Paths + +The `lfsStoragePaths` input sets the `LFS_STORAGE_PATHS` environment variable. How these paths +are interpreted depends on the agent: + +| Agent | Path format | +|-------|------------| +| elastic-git-storage | Local paths, `webdav://` URLs, `rclone://` remotes | +| lfs-folderstore | Local directory paths | +| Custom | Agent-specific | + +## Inputs Reference + +| Input | Description | +|-------|-------------| +| `lfsTransferAgent` | Agent name (e.g., `elastic-git-storage`) or path to executable. Append `@version` for release pinning. | +| `lfsTransferAgentArgs` | Additional arguments passed to the agent | +| `lfsStoragePaths` | Semicolon-separated storage paths (set as `LFS_STORAGE_PATHS` env var) |