diff --git a/.gitignore b/.gitignore index 6fd85bcf9..a6db66213 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,11 @@ bazel-* .aswb /.bazelrc.user MODULE.bazel.lock +# .NET +bin/ +obj/ +*.user +[Nn]upkg/ +.vs/ +*.suo +TestResults/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..62172e547 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,141 @@ +# CLAUDE.md — Copybara .NET Port + +This file orients Claude (and humans) working in this repository. It describes +what the project is, the state of the C# port, the conventions to follow, and +where things live. + +## What is Copybara? + +Copybara is a tool, originally built and open-sourced by Google, for +**transforming and moving code between repositories**. A typical use case is +keeping an internal (confidential) repository in sync with a public one: you +declare a *workflow* that reads changes from an *origin*, applies a chain of +*transformations* (move files, replace strings, scrub metadata, etc.), and +writes the result to a *destination*, recording state as a label in the +destination commit message so the process is stateless and reproducible. + +Configuration is written in **Starlark** (the Python-like dialect used by +Bazel), in a file conventionally named `copy.bara.sky`. The CLI exposes +subcommands like `migrate`, `info`, `validate`, and `version`. + +Key domain concepts: + +- **Origin** — where code is read from (git, GitHub, Gerrit, Mercurial, a + local folder, remote files, …). +- **Destination** — where code is written to (git, GitHub PR, a local folder, …). +- **Workflow** — ties an origin, destination, authoring, and a list of + transformations together. Runs in a mode (`SQUASH`, `ITERATIVE`, `CHANGE_REQUEST`). +- **Transformation** — a reversible (where possible) operation on the checkout + (`core.replace`, `core.move`, `core.copy`, metadata scrubbers, patch, …). +- **Revision / Change** — a point-in-time state of an origin (e.g. a git SHA) + and its metadata (author, message, labels, timestamp). +- **Glob** — include/exclude path matcher used pervasively to select files. +- **Authoring** — how author identity is mapped from origin to destination + (pass-thru, allowlist, overwrite). +- **Console** — abstraction over terminal I/O (ANSI, log, file, prompts). + +## Repository layout + +``` +/ Root of the .NET port +├── CLAUDE.md This file +├── TODO.md Detailed, living port plan / work breakdown +├── README.md Upstream Copybara readme (kept as-is) +├── LICENSE Apache 2.0 +├── Copybara.sln .NET solution +├── src/ +│ ├── Copybara.Common/ Guava-like collections, Preconditions, helpers +│ ├── Starlark/ Port of net.starlark.java (Starlark interpreter) +│ ├── Copybara.Core/ Port of com.google.copybara.* (the engine) +│ └── Copybara.Cli/ The .NET tool entry point (command `copybara`) +├── tests/ +│ └── Copybara.Tests/ xUnit tests (ports of javatests where practical) +└── java/ The ORIGINAL Java/Bazel implementation (reference only) + ├── com/google/copybara/... ~90k LOC — the engine + ├── third_party/bazel/.../net/starlark ~31k LOC — Starlark interpreter + ├── javatests/ Original test suite + ├── BUILD, MODULE.bazel, ... Original Bazel build + └── docs/, scripts/, ci/, ... +``` + +**The `java/` directory is the source of truth for behavior.** When porting a +class, open the corresponding Java file under `java/com/google/copybara/…` (or +`java/third_party/bazel/main/java/net/starlark/…` for the interpreter) and +mirror its behavior. Do not delete or modify files under `java/`. + +## Port goals + +1. **Target .NET 10**, C# latest language version, nullable reference types on. +2. **Ship as a .NET tool** (`dotnet tool install --global Copybara`) that + exposes a `copybara` command with the same CLI surface as upstream. +3. **Preserve behavior and config compatibility**: existing `copy.bara.sky` + configs should work. This requires a faithful Starlark interpreter. +4. **Use LibGit2Sharp** (NuGet package id: `LibGit2Sharp`) for git plumbing + instead of shelling out to `git` where practical; fall back to invoking the + `git` binary only where LibGit2Sharp lacks a feature (e.g. some fetch/push + refspec behaviors, credential helpers). +5. **Idiomatic C#**: PascalCase members, properties over getters where natural, + `async`/`Task` for I/O-bound repo operations, records for value types. +6. **Faithful but not slavish**: keep class/relationship structure recognizable + against the Java source, but drop Java-isms (checked exceptions, builder + boilerplate where records/init suffice, Guava types where BCL equivalents fit). + +## Java → C# mapping conventions + +Follow these consistently so the port stays coherent across contributors/agents. + +| Java / library | C# / .NET | +|----------------------------------------|-----------------------------------------------------------------| +| `com.google.copybara.foo` | namespace `Copybara.Foo` | +| `net.starlark.java.eval` | namespace `Starlark.Eval` (project `Starlark`) | +| `ImmutableList` | `ImmutableArray` (return as `IReadOnlyList` in APIs) | +| `ImmutableMap` | `ImmutableDictionary` / `IReadOnlyDictionary` | +| `ImmutableSet` | `ImmutableHashSet` / `IReadOnlySet` | +| `ImmutableListMultimap` | `Copybara.Common.ImmutableListMultimap` (helper type) | +| `Optional` | nullable (`T?`) for refs; `T?`/`Nullable` for structs | +| `Preconditions.checkNotNull/State` | `Copybara.Common.Preconditions` helpers | +| Guava `Splitter`/`Joiner`/`CharMatcher`| BCL `string.Split`/`string.Join` + helpers in `Copybara.Common` | +| checked exceptions (`throws`) | plain exceptions; document in XML ``; no `throws` | +| `@StarlarkBuiltin` / `@StarlarkMethod` | `[StarlarkBuiltin]` / `[StarlarkMethod]` attributes | +| `StarlarkValue` | `IStarlarkValue` marker interface | +| JGit | `LibGit2Sharp` (+ `git` binary fallback via a `GitEnv` runner) | +| JCommander (`@Parameter`) | custom lightweight arg parser in `Copybara.Cli` (see TODO) | +| Flogger `FluentLogger` | `Microsoft.Extensions.Logging.ILogger` | +| gson | `System.Text.Json` | +| re2j | .NET `System.Text.RegularExpressions` (note: not RE2 semantics) | +| `java.nio.file.Path` | `string` paths + `System.IO`; a `CheckoutPath` domain type | +| JUnit + Truth | xUnit + FluentAssertions | + +Notes / gotchas: + +- **Regex: use the native .NET engine.** Upstream uses re2j, but this port uses + `System.Text.RegularExpressions` — an accepted deviation. The vast majority of + Copybara patterns behave identically; note any observed divergence in TODO. +- **Starlark is the critical path.** Almost every domain object is a + `StarlarkValue` with `@StarlarkMethod` fields, and the config loader executes + Starlark. The `Starlark` project must land before most `*Module` classes can + be meaningfully ported. See TODO for the phased approach. +- **File header:** keep the Apache 2.0 license header on ported files. + +## Build / test + +``` +dotnet build Copybara.sln +dotnet test Copybara.sln +# Run the tool locally: +dotnet run --project src/Copybara.Cli -- migrate copy.bara.sky +# Pack as a global tool: +dotnet pack src/Copybara.Cli -c Release +``` + +## Status + +**Source port complete.** Every package under `java/com/google/copybara/**` plus +the vendored `net.starlark.java` interpreter has been ported to C# (~666 files, +~98.5k LOC). The whole solution builds with 0 warnings / 0 errors and the test +suite passes. Remaining work is *integration*, not source-porting — chiefly +wiring `ModuleSupplier` to register the modules/options so `copybara migrate` +runs a real `copy.bara.sky` end-to-end. **TODO.md is the living work breakdown** +— consult it for exactly what is done and what integration follow-ups remain. + + diff --git a/Copybara.slnx b/Copybara.slnx new file mode 100644 index 000000000..8552873eb --- /dev/null +++ b/Copybara.slnx @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 000000000..f537548f7 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,14 @@ + + + net10.0 + latest + enable + enable + false + false + Copybara .NET Port + Copybara + https://github.com/theolivenbaum/copybara + Apache-2.0 + + diff --git a/README.md b/README.md index f43724009..28326a8e2 100644 --- a/README.md +++ b/README.md @@ -1,265 +1,107 @@ -# Copybara +

+ Copybara logo +

-*A tool for transforming and moving code between repositories.* +# Copybara for .NET -Copybara is a tool used internally at Google. It transforms and moves code between repositories. +*A tool for transforming and moving code between repositories — a C# / .NET 10 +port of [Google Copybara](https://github.com/google/copybara).* -Often, source code needs to exist in multiple repositories, and Copybara allows you to transform -and move source code between these repositories. A common case is a project that involves -maintaining a confidential repository and a public repository in sync. +Copybara transforms and moves source code between repositories. A common use +case is keeping an internal (confidential) repository in sync with a public +one: you declare a *workflow* that reads changes from an **origin**, applies a +chain of **transformations** (move files, replace strings, scrub metadata, …), +and writes the result to a **destination**, recording state in the destination +commit message so the process is stateless and reproducible. -Copybara requires you to choose one of the repositories to be the authoritative repository, so that -there is always one source of truth. However, the tool allows contributions to any repository, and -any repository can be used to cut a release. - -The most common use case involves repetitive movement of code from one repository to another. -Copybara can also be used for moving code once to a new repository. - -Examples uses of Copybara include: - - - Importing sections of code from a confidential repository to a public repository. - - - Importing code from a public repository to a confidential repository. - - - Importing a change from a non-authoritative repository into the authoritative repository. When - a change is made in the non-authoritative repository (for example, a contributor in the public - repository), Copybara transforms and moves that change into the appropriate place in the - authoritative repository. Any merge conflicts are dealt with in the same way as an out-of-date - change within the authoritative repository. - -One of the main features of Copybara is that it is stateless, or more specifically, that it stores -the state in the destination repository (As a label in the commit message). This allows several -users (or a service) to use Copybara for the same config/repositories and get the same result. - -Currently, the only supported type of repository is Git. Copybara is also able -to read from Mercurial repositories, but the feature is still experimental. -The extensible architecture allows adding bespoke origins and destinations -for almost any use case. -Official support for other repositories types will be added in the future. - -## Example +Configuration is written in **Starlark** (the Python-like dialect used by +Bazel) in a file conventionally named `copy.bara.sky`. ```python core.workflow( name = "default", origin = git.github_origin( - url = "https://github.com/google/copybara.git", - ref = "master", + url = "https://github.com/google/copybara.git", + ref = "master", ), destination = git.destination( url = "file:///tmp/foo", ), - - # Copy everything but don't remove a README_INTERNAL.txt file if it exists. + # Copy everything but don't remove README_INTERNAL.txt if it exists. destination_files = glob(["third_party/copybara/**"], exclude = ["README_INTERNAL.txt"]), - authoring = authoring.pass_thru("Default email "), transformations = [ core.replace( - before = "//third_party/bazel/bashunit", - after = "//another/path:bashunit", - paths = glob(["**/BUILD"])), - core.move("", "third_party/copybara") + before = "//third_party/bazel/bashunit", + after = "//another/path:bashunit", + ), + core.move("foo/bar", "baz/bar"), ], ) ``` -Run: - -```shell -$ (mkdir /tmp/foo ; cd /tmp/foo ; git init --bare) -$ copybara copy.bara.sky -``` - -## Getting Started using Copybara - -The easiest way to start is with weekly "snapshot" releases, that include pre-built a binary. -Note that these are released automatically without any manual testing, version compatibility or correctness guarantees. - -Choose a release from https://github.com/google/copybara/releases. - -### Building from Source - -To use an unreleased version of copybara, so you need to compile from HEAD. -In order to do that, you need to do the following: - - * [Install JDK 11](https://www.oracle.com/java/technologies/downloads/#java11). - * [Install Bazel](https://bazel.build/install). - * Clone the copybara source locally: - * `git clone https://github.com/google/copybara.git` - * Build: - * `bazel build //java/com/google/copybara` - * `bazel build //java/com/google/copybara:copybara_deploy.jar` to create an executable uberjar. - * Tests: `bazel test //...` if you want to ensure you are not using a broken version. Note that - certain tests require the underlying tool to be installed(e.g. Mercurial, Quilt, etc.). It is - fine to skip those tests if your Pull Request is unrelated to those modules (And our CI will - run all the tests anyway). - -### System packages - -These packages can be installed using the appropriate package manager for your -system. - -#### Arch Linux - - * [`aur/copybara-git`][install/archlinux/aur-git] - -[install/archlinux/aur-git]: https://aur.archlinux.org/packages/copybara-git "Copybara on the AUR" - -### Using Intellij with Bazel plugin - -If you use Intellij and the Bazel plugin, use this project configuration: - -``` -directories: - copybara/integration - java/com/google/copybara - javatests/com/google/copybara - third_party - -targets: - //copybara/integration/... - //java/com/google/copybara/... - //javatests/com/google/copybara/... - //third_party/... -``` - -Note: configuration files can be stored in any place, even in a local folder. -We recommend using a VCS (like git) to store them; treat them as source code. - -### Using pre-built Copybara in Bazel - -If using a weekly snapshot release, install Copybara as follows: - -1. Copybara ships with class files with version 65.0, so it must be run with Java Runtime 21 or greater. Add to your `.bazelrc` file: `run --java_runtime_version=remotejdk_21` -2. Use `http_jar` to download the release artifact. - - In WORKSPACE: `load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar")` - - In MODULE.bazel: `http_jar = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar")` -3. In WORKSPACE or MODULE.bazel, fill in the `[version]` placeholder: - ```starlark - http_jar( - name = "com_github_google_copybara", - # Fill in from https://github.com/google/copybara/releases/download/[version]/copybara_deploy.jar.sha256 - # sha256 = "", - urls = ["https://github.com/google/copybara/releases/download/[version]/copybara_deploy.jar"], - ) - ``` -4. In any BUILD file (perhaps `/tools/BUILD.bazel`) declare the `java_binary`: - ```starlark - load("@rules_java//java:java_binary.bzl", "java_binary") - java_binary( - name = "copybara", - main_class = "com.google.copybara.Main", - runtime_deps = ["@com_github_google_copybara//jar"], - ) - ``` -5. Use that target with `bazel run`, for example `bazel run //tools:copybara -- migrate copy.bara.sky` - -### Building Copybara from Source as an external Bazel repository - -There are convenience macros defined for all of Copybara's dependencies. Add the -following code to your `WORKSPACE` file, replacing `{{ sha256sum }}` and -`{{ commit }}` as necessary. - -```bzl -http_archive( - name = "com_github_google_copybara", - sha256 = "{{ sha256sum }}", - strip_prefix = "copybara-{{ commit }}", - url = "https://github.com/google/copybara/archive/{{ commit }}.zip", -) - -load("@com_github_google_copybara//:repositories.bzl", "copybara_repositories") - -copybara_repositories() - -load("@com_github_google_copybara//:repositories.maven.bzl", "copybara_maven_repositories") - -copybara_maven_repositories() - -load("@com_github_google_copybara//:repositories.go.bzl", "copybara_go_repositories") - -copybara_go_repositories() -``` - -You can then build and run the Copybara tool from within your workspace: +## About this repository -```sh -bazel run @com_github_google_copybara//java/com/google/copybara -- -``` +This repository is a **port of Copybara from Java to C# targeting .NET 10**, +distributed as a **.NET global tool** named `copybara`. -### Using Docker to build and run Copybara +- The original Java/Bazel implementation is preserved, unmodified, under + [`java/`](java/) and is the reference for behavior. Its original README lives + at [`java/README.md`](java/README.md). +- The C# port lives under [`src/`](src/) and [`tests/`](tests/). +- [`CLAUDE.md`](CLAUDE.md) documents the architecture, porting conventions, and + Java→C# mappings. [`TODO.md`](TODO.md) is the living work breakdown and status. -*NOTE: Docker use is currently experimental, and we encourage feedback or contributions.* +> **Status:** the port is a work in progress. See [`TODO.md`](TODO.md) for what +> is implemented and what remains. -You can build copybara using Docker like so +## Layout -```sh -docker build --rm -t copybara . ``` - -Once this has finished building, you can run the image like so from the root of -the code you are trying to use Copybara on: - -```sh -docker run -it -v "$(pwd)":/usr/src/app copybara help +src/ + Copybara.Common/ Guava-like helpers (Preconditions, ImmutableListMultimap) + Starlark/ Port of the Starlark interpreter (net.starlark.java) + Copybara.Core/ The engine (origins, destinations, transformations, git, …) + Copybara.Cli/ The `copybara` .NET tool entry point +tests/ + Copybara.Tests/ xUnit tests +java/ The original Java/Bazel implementation (reference only) ``` -#### Environment variables +## Building -In addition to passing cmd args to the container, you can also set the following -environment variables as an alternative: -* `COPYBARA_SUBCOMMAND=migrate` - * allows you to change the command run, defaults to `migrate` -* `COPYBARA_CONFIG=copy.bara.sky` - * allows you to specify a path to a config file, defaults to root `copy.bara.sky` -* `COPYBARA_WORKFLOW=default` - * allows you to specify the workflow to run, defaults to `default` -* `COPYBARA_SOURCEREF=''` - * allows you to specify the sourceref, defaults to none -* `COPYBARA_OPTIONS=''` - * allows you to specify options for copybara, defaults to none +Requires the **.NET 10 SDK**. -```sh -docker run \ - -e COPYBARA_SUBCOMMAND='validate' \ - -e COPYBARA_CONFIG='other.config.sky' \ - -v "$(pwd)":/usr/src/app \ - -it copybara +```bash +dotnet build Copybara.slnx +dotnet test Copybara.slnx ``` -#### Git Config and Credentials +## Running -There are a number of ways by which to share your git config and ssh credentials -with the Docker container, an example is below: +```bash +# From source: +dotnet run --project src/Copybara.Cli -- migrate copy.bara.sky -```sh -docker run \ - -v ~/.gitconfig:/root/.gitconfig:ro \ - -v ~/.ssh:/root/.ssh \ - -v ${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK} -e SSH_AUTH_SOCK - -v "$(pwd)":/usr/src/app \ - -it copybara +# Packaged as a global tool: +dotnet pack src/Copybara.Cli -c Release +dotnet tool install --global --add-source src/Copybara.Cli/nupkg Copybara +copybara migrate copy.bara.sky ``` -## Documentation - -We are still working on the documentation. Here are some resources: - - * [Reference documentation](docs/reference.md) - * [Examples](docs/examples.md) - * [Tutorial on how to get started](https://blog.kubesimplify.com/moving-code-between-git-repositories-with-copybara) - -## Contact us - -If you have any questions about how Copybara works, please contact us at our -[mailing list](https://groups.google.com/forum/#!forum/copybara-discuss). +## Notable porting decisions -## Optional tips +- **Git** operations use the [`LibGit2Sharp`](https://www.nuget.org/packages/LibGit2Sharp) + NuGet package, with a fallback to invoking the `git` binary for features it + does not cover. +- **Regular expressions** use the native .NET regex engine + (`System.Text.RegularExpressions`) rather than RE2. This is an accepted + deviation from upstream (which uses re2j); the vast majority of Copybara + patterns behave identically. +- Starlark configuration semantics are preserved by porting the Bazel Starlark + interpreter to C# (see `src/Starlark`). -* If you want to see the test errors in Bazel, instead of having to `cat` the - logs, add this line to your `~/.bazelrc`: +## License - ``` - test --test_output=streamed - ``` +Apache License 2.0 — see [`LICENSE`](LICENSE). Copybara is a trademark of +Google LLC; this is an independent port. diff --git a/TODO.md b/TODO.md new file mode 100644 index 000000000..fbabca0fb --- /dev/null +++ b/TODO.md @@ -0,0 +1,188 @@ +# TODO — Copybara .NET 10 Port + +Living work breakdown. Status legend: ✅ done · 🚧 in progress · ⬜ pending · +🔬 needs investigation. + +The port is large (~90k LOC engine + ~31k LOC Starlark interpreter). It is +organized in dependency order: foundation first, then the Starlark interpreter, +then the domain modules that depend on both. + +--- + +## Phase 0 — Project setup + +- ✅ Move original Java/Bazel tree into `java/` (reference only). +- ✅ Author `CLAUDE.md` and `TODO.md`. +- ✅ Create solution + project scaffold (`Copybara.slnx`): + - `src/Copybara.Common`, `src/Starlark`, `src/Copybara.Core`, + `src/Copybara.Cli` (`PackAsTool=true`, verified `dotnet pack` produces a tool + package), `tests/Copybara.Tests`. +- ✅ Wire NuGet deps: `LibGit2Sharp`, `Microsoft.Extensions.Logging`, xUnit, + FluentAssertions (`System.Collections.Immutable`/`System.Text.Json` are in-box). +- ✅ `.gitignore` for .NET (`bin/`, `obj/`, `*.user`, `nupkg/`). +- ✅ Whole solution builds clean (0 warnings/0 errors); 8 smoke tests pass. +- ⬜ CI: `dotnet build` + `dotnet test` GitHub Action. + +## Phase 1 — Foundation (`Copybara.Common` + `Copybara.Core` primitives) + +No Starlark dependency. Port these first; they unblock everything else. + +- ✅ `Copybara.Common`: `Preconditions`, `ImmutableListMultimap` + builder. + (String helpers added ad hoc where needed.) +- ✅ `exception/` → `Copybara.Exceptions` — all 13 classes ported with correct + hierarchy; `ValidationException.CheckCondition` handles printf-style `%s` + format strings via an internal `%`→`{}` translator. +- ✅ `util/` core (subset) → `Copybara.Util`: `ExitCode`, full `Glob` subsystem + (`Glob`/`GlobAtom`/`SequenceGlob`/`ReadablePathMatcher`/`GlobPathMatcher`/ + `IPathMatcher`), `FileUtil`, `DirFactory`, `Identity`, `TablePrinter`, + `CommandOutput(WithStatus)`, `CommandRunner` + shell stack. + - ⬜ Still TODO in util: `DiffUtil`, `CommandLineDiffUtil`, `MergeImportTool`, + `AutoPatchUtil`, `ConsistencyFile`, `ApplyDestinationPatch`, `RenameDetector`, + `ScpUtil`, `OriginUtil`, `RepositoryUtil`, `EnumMapConverter`. +- ✅ `util/console/` → `Copybara.Util.Console` — all 17 files (`Console`, + `AnsiConsole`, `LogConsole`, `FileConsole`, `NoPromptConsole`, + `CapturingConsole`, `MultiplexingConsole`, `PrefixConsole`, `Consoles`, …). +- ✅ `revision/` → `Copybara.Revision`: `IRevision`, `Change`, `Changes`, `OriginRef`. +- ✅ `authoring/` → `Copybara.Authoring`: `Author`, `AuthorParser`, `Authoring`, + `InvalidAuthorException` (as Starlark values). +- ✅ `templatetoken/` → `Copybara.TemplateToken`. +- ⬜ `profiler/` → `Copybara.Profiler`. + +## Phase 2 — Starlark interpreter (`src/Starlark`) + +Port of `java/third_party/bazel/main/java/net/starlark/java`. This is a +self-contained interpreter and the critical dependency for config loading. +Sub-packages: `annot`, `syntax` (lexer/parser/AST), `eval` (values, evaluator, +builtins), `lib` (json, proto, etc.), `spelling`. + +- ✅ `annot/` — `[StarlarkBuiltin]`, `[StarlarkMethod]`, `[Param]`, `[ParamType]`. +- ✅ `syntax/` — `Lexer`, `Parser`, all AST node types, `Location`, `TokenKind`, + `FileOptions`, `SyntaxError`, `NodeVisitor`, `StarlarkFile`/`Program`, plus the + resolver/type-system subset (`Resolver`, `Types`, `StarlarkType`, `TypeChecker`, + `TypeTagger`, `TypeConstructor`, `TypeContext`). +- 🚧 `eval/` — DONE: value types (`StarlarkInt/Float/List/Tuple/Dict/RangeList`, + `Sequence`), `Mutability`, `StarlarkSemantics`, `Printer`, `Module`, + `StarlarkThread`, `EvalUtils`, `StarlarkValue`/`NoneType`/`EvalException`/ + `Starlark` helpers, callable interfaces. + - ✅ Evaluator + dispatch: `Eval` tree-walker, reflective dispatch + (`CallUtils`/`MethodDescriptor`/`ParamDescriptor`/`BuiltinFunction`/ + `StarlarkFunction`) mapping `[StarlarkMethod]`→calls, `MethodLibrary` + + `StringModule` builtins, and `Starlark.ExecFile`/`Eval`/`Call` drivers. + Parse→resolve→execute of real Starlark works (functions, comprehensions, + closures, builtins, string methods) — verified by xUnit `StarlarkEvalTests`. + - ⬜ Still deferred: `StarlarkSet` + its EvalUtils operator branches, `float` + builtin, flag-guarded params, dynamic arg/return type-checking. +- ✅ `spelling/` — `SpellChecker`. +- ⬜ `lib/json` — `Json` module (interop with `System.Text.Json`). +- ⬜ Reflection strategy: reflect over `[StarlarkMethod]`/`[Param]` at startup, + cached per type (source generators a later optimization). +- 🔬 Validate with upstream Starlark eval tests once the evaluator lands. + +## Phase 3 — Config model & core module (needs Phases 1–2) + +- ✅ `config/` → `Copybara.Config`: `Config`, `ConfigFile`, `PathBasedConfigFile`, + `MapConfigFile`, `IMigration`, `ConfigValidator`, `SkylarkParser` (wired to the + ported interpreter), `ConfigLoader`/`IConfigLoaderProvider`, `ILabelsAwareModule`, + `ConfigWithDependencies`. +- ✅ Top-level engine types: `IOption`/`Options`, `GeneralOptions`, `ModuleSet`/ + `ModuleSupplier`, `CoreModule`/`CoreGlobal`, `Workflow` (+ non-generic + `Workflow.Create` factory), `WorkflowOptions`, `WorkflowMode`, `WorkflowRunHelper`, + `IMigration`, `IOrigin`/`IDestination`, `ITransformation`, `TransformWork`, + `TransformResult`, `CheckoutPath`, `CheckoutFileSystem`, `Metadata`, `Info`, + `MigrationInfo`, plus `profiler/`, `effect/`, `monitor/`, `action/`, `approval/`, + `treestate/`, `version/`. + +## Phase 4 — Transformations (`transform/`) + +- ✅ `Replace`, `CopyOrMove`/`Remove`, `Sequence`, `ExplicitReversal`, + `IReversibleFunction`, `SkylarkConsole`, `FilterReplace`, `VerifyMatch`, + `TodoReplace`, `SkylarkTransformation`, `transform/metadata/*`, `transform/debug/*`. +- ⬜ `transform/patch/*` (needs DiffUtil/patch tooling). + +## Phase 5 — Git support (`git/`) — uses LibGit2Sharp / git CLI + +Largest single module (~175 files). Port in slices: + +- ✅ Core plumbing: `GitRepository` (git-CLI-backed via `CommandRunner`, faithful + to upstream which shells out), `GitRevision`, `GitRepoType`, `GitEnvironment`, + `GitCredential`, `Refspec`, `FetchResult`, `MergeResult`, `IntegrateLabel`, + `SameGitTree`, exceptions. +- ✅ Origin/Destination base: `GitOrigin`, `GitDestination`, `GitDestinationReader`, + `ChangeReader`, `GitVisitorUtil`, `Mirror`, `GitIntegrateChanges`, options, write hooks. +- ✅ `GitModule` (the `git` Starlark module — all 19 git.* factories wired). +- ✅ GitHub: `github/api` client (~50 files) + providers (`GitHubPrOrigin`, + `GitHubPrDestination`, `GitHubEndPoint`, write hooks, approvals validators, `github/util`). +- ✅ Gerrit: `gerritapi` client (30) + providers (`GerritOrigin`/`GerritDestination`/`GerritEndpoint`). +- ✅ GitLab: `gitlab/api` client (18) + providers (`GitLabMrOrigin`/`GitLabMrDestination`). +- ✅ `git/version/` selectors. ✅ `hg/` (Mercurial). + +## Phase 6 — Other origins/destinations & modules + +- ✅ `folder/` (FolderOrigin/FolderDestination/FolderModule) — first ported origin/destination pair. +- ✅ `remotefile/`, `archive/` (zip/tar/gzip; xz/bz2 = TODO), `hashing/`, + `http/` (HttpClient), `format/` (buildifier), `buildozer/`, `toml/`, `json/`, + `xml/`, `html/`, `re2/`, `credentials/`, `approval/`, `action/`, `treestate/`, + `monitor/`, `effect/`, `checks/` (minimal stub). +- ✅ `hg/` (Mercurial), `go/`, `rust/`, `python/`, `tsjs/` (npm). +- ✅ `feedback/`, `checks/`, `regenerate/`, `onboard/`, `configgen/`, + `doc/` (reflection-based reference generator), `transform/patch/`. +- ✅ `starlark/StarlarkUtil`, `archive/util`. + +## Source port: COMPLETE + +Every source package under `java/com/google/copybara/**` and the vendored +`net.starlark.java` interpreter has a C# counterpart. ~666 C# files / ~98.5k LOC. +Whole solution builds 0 warnings / 0 errors; tests pass. + +Intentionally NOT ported (superseded/obsolete): `jcommander/*` converters/validators +(replaced by the custom `Copybara.Cli.ArgParser`). + +### Remaining integration / follow-up work (not source-porting) +- **Wire `ModuleSupplier.GetModules()`** to register the ported Starlark modules + (`Core`, `git`, `folder`, `format`, `http`, `hashing`, `archive`, `remotefile`, + `toml`/`json`/`xml`/`html`/`re2`, `go`/`rust`/`python`/`npm`, `credentials`, …) + and `NewOptions()` to register every `IOption`. Currently stubbed empty — this is + what makes `copybara migrate` load a real `copy.bara.sky` end-to-end. +- **Wire CLI commands**: `RegenerateCmd`/`OnboardCmd`/`GeneratorCmd` engines exist in + Core with `Run(...)` entry points; add thin `ICopybaraCmd` adapters in `Copybara.Cli`. +- **Archive xz/bz2** (`TAR_XZ`/`TAR_BZ2`) need a codec (no in-box option) — currently + throw with a `TODO(port)`. +- **Expand test coverage**: port high-value suites from `java/javatests/` and add an + end-to-end folder→folder `migrate` smoke test. +- Verify a few `structField`/reflective-dispatch edge cases against real configs. + +## Phase 7 — CLI (`src/Copybara.Cli`) + +- ✅ Arg parsing (custom `ArgParser` replacing JCommander, reading `[Flag]` + attributes off option objects), matching upstream flag names. +- ✅ `Main` orchestration (mirrors `Main.java`): console setup, logging config, + module set creation, command dispatch, exit codes, error handling. +- ✅ Commands: `MigrateCmd`, `InfoCmd`, `ValidateCmd` (+ version/help). +- ⬜ Commands not yet ported: `RegenerateCmd`, `OnboardCmd`/`GeneratorCmd`. +- ✅ `PackAsTool` + package icon/readme. ⬜ `build-data` version embedding. + +## Phase 8 — Tests, docs, polish + +- ⬜ Port high-value tests from `javatests/` (glob, author, replace, workflow, + git origin/destination round-trips). +- ⬜ End-to-end smoke test: folder→folder migrate with a `core.replace`. +- ⬜ Reference doc generation parity (optional). +- ⬜ Performance pass; RE2 semantics decision. + +--- + +## Cross-cutting decisions / open questions + +- ✅ **RE2 vs .NET Regex** — DECIDED: use the native .NET regex engine + (`System.Text.RegularExpressions`). Accepted deviation from upstream re2j. + Flag any divergences observed during porting/testing. +- 🔬 **Async vs sync** — repo I/O is naturally async in .NET; upstream is + blocking. Decide per-boundary; keep the engine mostly synchronous initially to + stay close to the source, use async only at process/network edges. +- 🔬 **Path handling** — Java uses `java.nio.file.Path` + Jimfs in tests. Use + `string` + `System.IO`, and an in-memory filesystem abstraction for tests if + needed. +- 🔬 **git CLI dependency** — LibGit2Sharp covers most, but shallow fetch, + some refspec and credential-helper behaviors may still require the `git` + binary. Keep `GitEnv`/`CommandRunner` available as a fallback. + diff --git a/assets/icon-256.png b/assets/icon-256.png new file mode 100644 index 000000000..9a068e0cb Binary files /dev/null and b/assets/icon-256.png differ diff --git a/assets/icon.png b/assets/icon.png new file mode 100644 index 000000000..e56ce346e Binary files /dev/null and b/assets/icon.png differ diff --git a/global.json b/global.json new file mode 100644 index 000000000..512142d2b --- /dev/null +++ b/global.json @@ -0,0 +1,6 @@ +{ + "sdk": { + "version": "10.0.100", + "rollForward": "latestFeature" + } +} diff --git a/.bazelproject b/java/.bazelproject similarity index 100% rename from .bazelproject rename to java/.bazelproject diff --git a/.bazelrc b/java/.bazelrc similarity index 100% rename from .bazelrc rename to java/.bazelrc diff --git a/.docker/copybara b/java/.docker/copybara similarity index 100% rename from .docker/copybara rename to java/.docker/copybara diff --git a/.dockerignore b/java/.dockerignore similarity index 100% rename from .dockerignore rename to java/.dockerignore diff --git a/BUILD b/java/BUILD similarity index 100% rename from BUILD rename to java/BUILD diff --git a/Dockerfile b/java/Dockerfile similarity index 100% rename from Dockerfile rename to java/Dockerfile diff --git a/MODULE.bazel b/java/MODULE.bazel similarity index 100% rename from MODULE.bazel rename to java/MODULE.bazel diff --git a/java/README.md b/java/README.md new file mode 100644 index 000000000..f43724009 --- /dev/null +++ b/java/README.md @@ -0,0 +1,265 @@ +# Copybara + +*A tool for transforming and moving code between repositories.* + +Copybara is a tool used internally at Google. It transforms and moves code between repositories. + +Often, source code needs to exist in multiple repositories, and Copybara allows you to transform +and move source code between these repositories. A common case is a project that involves +maintaining a confidential repository and a public repository in sync. + +Copybara requires you to choose one of the repositories to be the authoritative repository, so that +there is always one source of truth. However, the tool allows contributions to any repository, and +any repository can be used to cut a release. + +The most common use case involves repetitive movement of code from one repository to another. +Copybara can also be used for moving code once to a new repository. + +Examples uses of Copybara include: + + - Importing sections of code from a confidential repository to a public repository. + + - Importing code from a public repository to a confidential repository. + + - Importing a change from a non-authoritative repository into the authoritative repository. When + a change is made in the non-authoritative repository (for example, a contributor in the public + repository), Copybara transforms and moves that change into the appropriate place in the + authoritative repository. Any merge conflicts are dealt with in the same way as an out-of-date + change within the authoritative repository. + +One of the main features of Copybara is that it is stateless, or more specifically, that it stores +the state in the destination repository (As a label in the commit message). This allows several +users (or a service) to use Copybara for the same config/repositories and get the same result. + +Currently, the only supported type of repository is Git. Copybara is also able +to read from Mercurial repositories, but the feature is still experimental. +The extensible architecture allows adding bespoke origins and destinations +for almost any use case. +Official support for other repositories types will be added in the future. + +## Example + +```python +core.workflow( + name = "default", + origin = git.github_origin( + url = "https://github.com/google/copybara.git", + ref = "master", + ), + destination = git.destination( + url = "file:///tmp/foo", + ), + + # Copy everything but don't remove a README_INTERNAL.txt file if it exists. + destination_files = glob(["third_party/copybara/**"], exclude = ["README_INTERNAL.txt"]), + + authoring = authoring.pass_thru("Default email "), + transformations = [ + core.replace( + before = "//third_party/bazel/bashunit", + after = "//another/path:bashunit", + paths = glob(["**/BUILD"])), + core.move("", "third_party/copybara") + ], +) +``` + +Run: + +```shell +$ (mkdir /tmp/foo ; cd /tmp/foo ; git init --bare) +$ copybara copy.bara.sky +``` + +## Getting Started using Copybara + +The easiest way to start is with weekly "snapshot" releases, that include pre-built a binary. +Note that these are released automatically without any manual testing, version compatibility or correctness guarantees. + +Choose a release from https://github.com/google/copybara/releases. + +### Building from Source + +To use an unreleased version of copybara, so you need to compile from HEAD. +In order to do that, you need to do the following: + + * [Install JDK 11](https://www.oracle.com/java/technologies/downloads/#java11). + * [Install Bazel](https://bazel.build/install). + * Clone the copybara source locally: + * `git clone https://github.com/google/copybara.git` + * Build: + * `bazel build //java/com/google/copybara` + * `bazel build //java/com/google/copybara:copybara_deploy.jar` to create an executable uberjar. + * Tests: `bazel test //...` if you want to ensure you are not using a broken version. Note that + certain tests require the underlying tool to be installed(e.g. Mercurial, Quilt, etc.). It is + fine to skip those tests if your Pull Request is unrelated to those modules (And our CI will + run all the tests anyway). + +### System packages + +These packages can be installed using the appropriate package manager for your +system. + +#### Arch Linux + + * [`aur/copybara-git`][install/archlinux/aur-git] + +[install/archlinux/aur-git]: https://aur.archlinux.org/packages/copybara-git "Copybara on the AUR" + +### Using Intellij with Bazel plugin + +If you use Intellij and the Bazel plugin, use this project configuration: + +``` +directories: + copybara/integration + java/com/google/copybara + javatests/com/google/copybara + third_party + +targets: + //copybara/integration/... + //java/com/google/copybara/... + //javatests/com/google/copybara/... + //third_party/... +``` + +Note: configuration files can be stored in any place, even in a local folder. +We recommend using a VCS (like git) to store them; treat them as source code. + +### Using pre-built Copybara in Bazel + +If using a weekly snapshot release, install Copybara as follows: + +1. Copybara ships with class files with version 65.0, so it must be run with Java Runtime 21 or greater. Add to your `.bazelrc` file: `run --java_runtime_version=remotejdk_21` +2. Use `http_jar` to download the release artifact. + - In WORKSPACE: `load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar")` + - In MODULE.bazel: `http_jar = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar")` +3. In WORKSPACE or MODULE.bazel, fill in the `[version]` placeholder: + ```starlark + http_jar( + name = "com_github_google_copybara", + # Fill in from https://github.com/google/copybara/releases/download/[version]/copybara_deploy.jar.sha256 + # sha256 = "", + urls = ["https://github.com/google/copybara/releases/download/[version]/copybara_deploy.jar"], + ) + ``` +4. In any BUILD file (perhaps `/tools/BUILD.bazel`) declare the `java_binary`: + ```starlark + load("@rules_java//java:java_binary.bzl", "java_binary") + java_binary( + name = "copybara", + main_class = "com.google.copybara.Main", + runtime_deps = ["@com_github_google_copybara//jar"], + ) + ``` +5. Use that target with `bazel run`, for example `bazel run //tools:copybara -- migrate copy.bara.sky` + +### Building Copybara from Source as an external Bazel repository + +There are convenience macros defined for all of Copybara's dependencies. Add the +following code to your `WORKSPACE` file, replacing `{{ sha256sum }}` and +`{{ commit }}` as necessary. + +```bzl +http_archive( + name = "com_github_google_copybara", + sha256 = "{{ sha256sum }}", + strip_prefix = "copybara-{{ commit }}", + url = "https://github.com/google/copybara/archive/{{ commit }}.zip", +) + +load("@com_github_google_copybara//:repositories.bzl", "copybara_repositories") + +copybara_repositories() + +load("@com_github_google_copybara//:repositories.maven.bzl", "copybara_maven_repositories") + +copybara_maven_repositories() + +load("@com_github_google_copybara//:repositories.go.bzl", "copybara_go_repositories") + +copybara_go_repositories() +``` + +You can then build and run the Copybara tool from within your workspace: + +```sh +bazel run @com_github_google_copybara//java/com/google/copybara -- +``` + +### Using Docker to build and run Copybara + +*NOTE: Docker use is currently experimental, and we encourage feedback or contributions.* + +You can build copybara using Docker like so + +```sh +docker build --rm -t copybara . +``` + +Once this has finished building, you can run the image like so from the root of +the code you are trying to use Copybara on: + +```sh +docker run -it -v "$(pwd)":/usr/src/app copybara help +``` + +#### Environment variables + +In addition to passing cmd args to the container, you can also set the following +environment variables as an alternative: +* `COPYBARA_SUBCOMMAND=migrate` + * allows you to change the command run, defaults to `migrate` +* `COPYBARA_CONFIG=copy.bara.sky` + * allows you to specify a path to a config file, defaults to root `copy.bara.sky` +* `COPYBARA_WORKFLOW=default` + * allows you to specify the workflow to run, defaults to `default` +* `COPYBARA_SOURCEREF=''` + * allows you to specify the sourceref, defaults to none +* `COPYBARA_OPTIONS=''` + * allows you to specify options for copybara, defaults to none + +```sh +docker run \ + -e COPYBARA_SUBCOMMAND='validate' \ + -e COPYBARA_CONFIG='other.config.sky' \ + -v "$(pwd)":/usr/src/app \ + -it copybara +``` + +#### Git Config and Credentials + +There are a number of ways by which to share your git config and ssh credentials +with the Docker container, an example is below: + +```sh +docker run \ + -v ~/.gitconfig:/root/.gitconfig:ro \ + -v ~/.ssh:/root/.ssh \ + -v ${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK} -e SSH_AUTH_SOCK + -v "$(pwd)":/usr/src/app \ + -it copybara +``` + +## Documentation + +We are still working on the documentation. Here are some resources: + + * [Reference documentation](docs/reference.md) + * [Examples](docs/examples.md) + * [Tutorial on how to get started](https://blog.kubesimplify.com/moving-code-between-git-repositories-with-copybara) + +## Contact us + +If you have any questions about how Copybara works, please contact us at our +[mailing list](https://groups.google.com/forum/#!forum/copybara-discuss). + +## Optional tips + +* If you want to see the test errors in Bazel, instead of having to `cat` the + logs, add this line to your `~/.bazelrc`: + + ``` + test --test_output=streamed + ``` diff --git a/ci.Dockerfile b/java/ci.Dockerfile similarity index 100% rename from ci.Dockerfile rename to java/ci.Dockerfile diff --git a/ci/ubuntu/continuous.cfg b/java/ci/ubuntu/continuous.cfg similarity index 100% rename from ci/ubuntu/continuous.cfg rename to java/ci/ubuntu/continuous.cfg diff --git a/ci/ubuntu/continuous.sh b/java/ci/ubuntu/continuous.sh similarity index 100% rename from ci/ubuntu/continuous.sh rename to java/ci/ubuntu/continuous.sh diff --git a/cloudbuild.sh b/java/cloudbuild.sh similarity index 100% rename from cloudbuild.sh rename to java/cloudbuild.sh diff --git a/cloudbuild.yaml b/java/cloudbuild.yaml similarity index 100% rename from cloudbuild.yaml rename to java/cloudbuild.yaml diff --git a/cloudbuild_setup.sh b/java/cloudbuild_setup.sh similarity index 100% rename from cloudbuild_setup.sh rename to java/cloudbuild_setup.sh diff --git a/copybara/integration/BUILD b/java/copybara/integration/BUILD similarity index 100% rename from copybara/integration/BUILD rename to java/copybara/integration/BUILD diff --git a/copybara/integration/reference_doc_test.sh b/java/copybara/integration/reference_doc_test.sh similarity index 100% rename from copybara/integration/reference_doc_test.sh rename to java/copybara/integration/reference_doc_test.sh diff --git a/copybara/integration/test-help.t b/java/copybara/integration/test-help.t similarity index 100% rename from copybara/integration/test-help.t rename to java/copybara/integration/test-help.t diff --git a/copybara/integration/tool_test.sh b/java/copybara/integration/tool_test.sh similarity index 100% rename from copybara/integration/tool_test.sh rename to java/copybara/integration/tool_test.sh diff --git a/docs/BUILD b/java/docs/BUILD similarity index 100% rename from docs/BUILD rename to java/docs/BUILD diff --git a/docs/examples.md b/java/docs/examples.md similarity index 100% rename from docs/examples.md rename to java/docs/examples.md diff --git a/docs/reference.md b/java/docs/reference.md similarity index 100% rename from docs/reference.md rename to java/docs/reference.md diff --git a/external/third_party/BUILD b/java/external/third_party/BUILD similarity index 100% rename from external/third_party/BUILD rename to java/external/third_party/BUILD diff --git a/external/third_party/jcommander.BUILD b/java/external/third_party/jcommander.BUILD similarity index 100% rename from external/third_party/jcommander.BUILD rename to java/external/third_party/jcommander.BUILD diff --git a/javatests/com/google/copybara/ActionMigrationTest.java b/java/javatests/com/google/copybara/ActionMigrationTest.java similarity index 100% rename from javatests/com/google/copybara/ActionMigrationTest.java rename to java/javatests/com/google/copybara/ActionMigrationTest.java diff --git a/javatests/com/google/copybara/BUILD b/java/javatests/com/google/copybara/BUILD similarity index 100% rename from javatests/com/google/copybara/BUILD rename to java/javatests/com/google/copybara/BUILD diff --git a/javatests/com/google/copybara/BaselinesWithoutLabelVisitorTest.java b/java/javatests/com/google/copybara/BaselinesWithoutLabelVisitorTest.java similarity index 100% rename from javatests/com/google/copybara/BaselinesWithoutLabelVisitorTest.java rename to java/javatests/com/google/copybara/BaselinesWithoutLabelVisitorTest.java diff --git a/javatests/com/google/copybara/ChangeMessageTest.java b/java/javatests/com/google/copybara/ChangeMessageTest.java similarity index 100% rename from javatests/com/google/copybara/ChangeMessageTest.java rename to java/javatests/com/google/copybara/ChangeMessageTest.java diff --git a/javatests/com/google/copybara/ChangesResponseTest.java b/java/javatests/com/google/copybara/ChangesResponseTest.java similarity index 100% rename from javatests/com/google/copybara/ChangesResponseTest.java rename to java/javatests/com/google/copybara/ChangesResponseTest.java diff --git a/javatests/com/google/copybara/CheckoutPathTest.java b/java/javatests/com/google/copybara/CheckoutPathTest.java similarity index 100% rename from javatests/com/google/copybara/CheckoutPathTest.java rename to java/javatests/com/google/copybara/CheckoutPathTest.java diff --git a/javatests/com/google/copybara/CommandEnvTest.java b/java/javatests/com/google/copybara/CommandEnvTest.java similarity index 100% rename from javatests/com/google/copybara/CommandEnvTest.java rename to java/javatests/com/google/copybara/CommandEnvTest.java diff --git a/javatests/com/google/copybara/CommandRunnerTest.java b/java/javatests/com/google/copybara/CommandRunnerTest.java similarity index 100% rename from javatests/com/google/copybara/CommandRunnerTest.java rename to java/javatests/com/google/copybara/CommandRunnerTest.java diff --git a/javatests/com/google/copybara/CoreReverseTest.java b/java/javatests/com/google/copybara/CoreReverseTest.java similarity index 100% rename from javatests/com/google/copybara/CoreReverseTest.java rename to java/javatests/com/google/copybara/CoreReverseTest.java diff --git a/javatests/com/google/copybara/CoreTransformTest.java b/java/javatests/com/google/copybara/CoreTransformTest.java similarity index 100% rename from javatests/com/google/copybara/CoreTransformTest.java rename to java/javatests/com/google/copybara/CoreTransformTest.java diff --git a/javatests/com/google/copybara/ExamplesTest.java b/java/javatests/com/google/copybara/ExamplesTest.java similarity index 100% rename from javatests/com/google/copybara/ExamplesTest.java rename to java/javatests/com/google/copybara/ExamplesTest.java diff --git a/javatests/com/google/copybara/InfoTest.java b/java/javatests/com/google/copybara/InfoTest.java similarity index 100% rename from javatests/com/google/copybara/InfoTest.java rename to java/javatests/com/google/copybara/InfoTest.java diff --git a/javatests/com/google/copybara/LabelFinderTest.java b/java/javatests/com/google/copybara/LabelFinderTest.java similarity index 100% rename from javatests/com/google/copybara/LabelFinderTest.java rename to java/javatests/com/google/copybara/LabelFinderTest.java diff --git a/javatests/com/google/copybara/MainArgumentsTest.java b/java/javatests/com/google/copybara/MainArgumentsTest.java similarity index 100% rename from javatests/com/google/copybara/MainArgumentsTest.java rename to java/javatests/com/google/copybara/MainArgumentsTest.java diff --git a/javatests/com/google/copybara/MainTest.java b/java/javatests/com/google/copybara/MainTest.java similarity index 100% rename from javatests/com/google/copybara/MainTest.java rename to java/javatests/com/google/copybara/MainTest.java diff --git a/javatests/com/google/copybara/MigrateCmdTest.java b/java/javatests/com/google/copybara/MigrateCmdTest.java similarity index 100% rename from javatests/com/google/copybara/MigrateCmdTest.java rename to java/javatests/com/google/copybara/MigrateCmdTest.java diff --git a/javatests/com/google/copybara/ReadConfigFromChangeWorkflowTest.java b/java/javatests/com/google/copybara/ReadConfigFromChangeWorkflowTest.java similarity index 100% rename from javatests/com/google/copybara/ReadConfigFromChangeWorkflowTest.java rename to java/javatests/com/google/copybara/ReadConfigFromChangeWorkflowTest.java diff --git a/javatests/com/google/copybara/ReferenceTest.java b/java/javatests/com/google/copybara/ReferenceTest.java similarity index 100% rename from javatests/com/google/copybara/ReferenceTest.java rename to java/javatests/com/google/copybara/ReferenceTest.java diff --git a/javatests/com/google/copybara/StarlarkDateTimeModuleTest.java b/java/javatests/com/google/copybara/StarlarkDateTimeModuleTest.java similarity index 100% rename from javatests/com/google/copybara/StarlarkDateTimeModuleTest.java rename to java/javatests/com/google/copybara/StarlarkDateTimeModuleTest.java diff --git a/javatests/com/google/copybara/StarlarkJsonModuleTest.java b/java/javatests/com/google/copybara/StarlarkJsonModuleTest.java similarity index 100% rename from javatests/com/google/copybara/StarlarkJsonModuleTest.java rename to java/javatests/com/google/copybara/StarlarkJsonModuleTest.java diff --git a/javatests/com/google/copybara/StarlarkRandomModuleTest.java b/java/javatests/com/google/copybara/StarlarkRandomModuleTest.java similarity index 100% rename from javatests/com/google/copybara/StarlarkRandomModuleTest.java rename to java/javatests/com/google/copybara/StarlarkRandomModuleTest.java diff --git a/javatests/com/google/copybara/StructModuleTest.java b/java/javatests/com/google/copybara/StructModuleTest.java similarity index 100% rename from javatests/com/google/copybara/StructModuleTest.java rename to java/javatests/com/google/copybara/StructModuleTest.java diff --git a/javatests/com/google/copybara/TransformWorkTest.java b/java/javatests/com/google/copybara/TransformWorkTest.java similarity index 100% rename from javatests/com/google/copybara/TransformWorkTest.java rename to java/javatests/com/google/copybara/TransformWorkTest.java diff --git a/javatests/com/google/copybara/WorkflowTest.java b/java/javatests/com/google/copybara/WorkflowTest.java similarity index 100% rename from javatests/com/google/copybara/WorkflowTest.java rename to java/javatests/com/google/copybara/WorkflowTest.java diff --git a/javatests/com/google/copybara/archive/ArchiveModuleTest.java b/java/javatests/com/google/copybara/archive/ArchiveModuleTest.java similarity index 100% rename from javatests/com/google/copybara/archive/ArchiveModuleTest.java rename to java/javatests/com/google/copybara/archive/ArchiveModuleTest.java diff --git a/javatests/com/google/copybara/archive/BUILD b/java/javatests/com/google/copybara/archive/BUILD similarity index 100% rename from javatests/com/google/copybara/archive/BUILD rename to java/javatests/com/google/copybara/archive/BUILD diff --git a/javatests/com/google/copybara/archive/util/ArchiveUtilTest.java b/java/javatests/com/google/copybara/archive/util/ArchiveUtilTest.java similarity index 100% rename from javatests/com/google/copybara/archive/util/ArchiveUtilTest.java rename to java/javatests/com/google/copybara/archive/util/ArchiveUtilTest.java diff --git a/javatests/com/google/copybara/archive/util/BUILD b/java/javatests/com/google/copybara/archive/util/BUILD similarity index 100% rename from javatests/com/google/copybara/archive/util/BUILD rename to java/javatests/com/google/copybara/archive/util/BUILD diff --git a/javatests/com/google/copybara/authoring/AuthorParserTest.java b/java/javatests/com/google/copybara/authoring/AuthorParserTest.java similarity index 100% rename from javatests/com/google/copybara/authoring/AuthorParserTest.java rename to java/javatests/com/google/copybara/authoring/AuthorParserTest.java diff --git a/javatests/com/google/copybara/authoring/AuthorTest.java b/java/javatests/com/google/copybara/authoring/AuthorTest.java similarity index 100% rename from javatests/com/google/copybara/authoring/AuthorTest.java rename to java/javatests/com/google/copybara/authoring/AuthorTest.java diff --git a/javatests/com/google/copybara/authoring/AuthoringTest.java b/java/javatests/com/google/copybara/authoring/AuthoringTest.java similarity index 100% rename from javatests/com/google/copybara/authoring/AuthoringTest.java rename to java/javatests/com/google/copybara/authoring/AuthoringTest.java diff --git a/javatests/com/google/copybara/buildozer/BUILD b/java/javatests/com/google/copybara/buildozer/BUILD similarity index 100% rename from javatests/com/google/copybara/buildozer/BUILD rename to java/javatests/com/google/copybara/buildozer/BUILD diff --git a/javatests/com/google/copybara/buildozer/BuildozerBatchTest.java b/java/javatests/com/google/copybara/buildozer/BuildozerBatchTest.java similarity index 100% rename from javatests/com/google/copybara/buildozer/BuildozerBatchTest.java rename to java/javatests/com/google/copybara/buildozer/BuildozerBatchTest.java diff --git a/javatests/com/google/copybara/buildozer/BuildozerCreateTest.java b/java/javatests/com/google/copybara/buildozer/BuildozerCreateTest.java similarity index 100% rename from javatests/com/google/copybara/buildozer/BuildozerCreateTest.java rename to java/javatests/com/google/copybara/buildozer/BuildozerCreateTest.java diff --git a/javatests/com/google/copybara/buildozer/BuildozerDeleteTest.java b/java/javatests/com/google/copybara/buildozer/BuildozerDeleteTest.java similarity index 100% rename from javatests/com/google/copybara/buildozer/BuildozerDeleteTest.java rename to java/javatests/com/google/copybara/buildozer/BuildozerDeleteTest.java diff --git a/javatests/com/google/copybara/buildozer/BuildozerModifyTest.java b/java/javatests/com/google/copybara/buildozer/BuildozerModifyTest.java similarity index 100% rename from javatests/com/google/copybara/buildozer/BuildozerModifyTest.java rename to java/javatests/com/google/copybara/buildozer/BuildozerModifyTest.java diff --git a/javatests/com/google/copybara/buildozer/BuildozerPrintExecutorTest.java b/java/javatests/com/google/copybara/buildozer/BuildozerPrintExecutorTest.java similarity index 100% rename from javatests/com/google/copybara/buildozer/BuildozerPrintExecutorTest.java rename to java/javatests/com/google/copybara/buildozer/BuildozerPrintExecutorTest.java diff --git a/javatests/com/google/copybara/checks/ApiCheckerTest.java b/java/javatests/com/google/copybara/checks/ApiCheckerTest.java similarity index 100% rename from javatests/com/google/copybara/checks/ApiCheckerTest.java rename to java/javatests/com/google/copybara/checks/ApiCheckerTest.java diff --git a/javatests/com/google/copybara/checks/BUILD b/java/javatests/com/google/copybara/checks/BUILD similarity index 100% rename from javatests/com/google/copybara/checks/BUILD rename to java/javatests/com/google/copybara/checks/BUILD diff --git a/javatests/com/google/copybara/config/BUILD b/java/javatests/com/google/copybara/config/BUILD similarity index 100% rename from javatests/com/google/copybara/config/BUILD rename to java/javatests/com/google/copybara/config/BUILD diff --git a/javatests/com/google/copybara/config/CapturingConfigFileTest.java b/java/javatests/com/google/copybara/config/CapturingConfigFileTest.java similarity index 100% rename from javatests/com/google/copybara/config/CapturingConfigFileTest.java rename to java/javatests/com/google/copybara/config/CapturingConfigFileTest.java diff --git a/javatests/com/google/copybara/config/MapConfigFileTest.java b/java/javatests/com/google/copybara/config/MapConfigFileTest.java similarity index 100% rename from javatests/com/google/copybara/config/MapConfigFileTest.java rename to java/javatests/com/google/copybara/config/MapConfigFileTest.java diff --git a/javatests/com/google/copybara/config/PathBasedConfigFileTest.java b/java/javatests/com/google/copybara/config/PathBasedConfigFileTest.java similarity index 100% rename from javatests/com/google/copybara/config/PathBasedConfigFileTest.java rename to java/javatests/com/google/copybara/config/PathBasedConfigFileTest.java diff --git a/javatests/com/google/copybara/config/ResolveDelegateConfigFileTest.java b/java/javatests/com/google/copybara/config/ResolveDelegateConfigFileTest.java similarity index 100% rename from javatests/com/google/copybara/config/ResolveDelegateConfigFileTest.java rename to java/javatests/com/google/copybara/config/ResolveDelegateConfigFileTest.java diff --git a/javatests/com/google/copybara/config/SkylarkParserTest.java b/java/javatests/com/google/copybara/config/SkylarkParserTest.java similarity index 100% rename from javatests/com/google/copybara/config/SkylarkParserTest.java rename to java/javatests/com/google/copybara/config/SkylarkParserTest.java diff --git a/javatests/com/google/copybara/config/SkylarkUtilTest.java b/java/javatests/com/google/copybara/config/SkylarkUtilTest.java similarity index 100% rename from javatests/com/google/copybara/config/SkylarkUtilTest.java rename to java/javatests/com/google/copybara/config/SkylarkUtilTest.java diff --git a/javatests/com/google/copybara/config/ValidationResultTest.java b/java/javatests/com/google/copybara/config/ValidationResultTest.java similarity index 100% rename from javatests/com/google/copybara/config/ValidationResultTest.java rename to java/javatests/com/google/copybara/config/ValidationResultTest.java diff --git a/javatests/com/google/copybara/config/base/BUILD b/java/javatests/com/google/copybara/config/base/BUILD similarity index 100% rename from javatests/com/google/copybara/config/base/BUILD rename to java/javatests/com/google/copybara/config/base/BUILD diff --git a/javatests/com/google/copybara/config/base/SkylarkUtilTest.java b/java/javatests/com/google/copybara/config/base/SkylarkUtilTest.java similarity index 100% rename from javatests/com/google/copybara/config/base/SkylarkUtilTest.java rename to java/javatests/com/google/copybara/config/base/SkylarkUtilTest.java diff --git a/javatests/com/google/copybara/configgen/BUILD b/java/javatests/com/google/copybara/configgen/BUILD similarity index 100% rename from javatests/com/google/copybara/configgen/BUILD rename to java/javatests/com/google/copybara/configgen/BUILD diff --git a/javatests/com/google/copybara/configgen/ConfigGenHeuristicsTest.java b/java/javatests/com/google/copybara/configgen/ConfigGenHeuristicsTest.java similarity index 100% rename from javatests/com/google/copybara/configgen/ConfigGenHeuristicsTest.java rename to java/javatests/com/google/copybara/configgen/ConfigGenHeuristicsTest.java diff --git a/javatests/com/google/copybara/credentials/BUILD b/java/javatests/com/google/copybara/credentials/BUILD similarity index 100% rename from javatests/com/google/copybara/credentials/BUILD rename to java/javatests/com/google/copybara/credentials/BUILD diff --git a/javatests/com/google/copybara/credentials/CredentialModuleTest.java b/java/javatests/com/google/copybara/credentials/CredentialModuleTest.java similarity index 100% rename from javatests/com/google/copybara/credentials/CredentialModuleTest.java rename to java/javatests/com/google/copybara/credentials/CredentialModuleTest.java diff --git a/javatests/com/google/copybara/credentials/TtlSecretTest.java b/java/javatests/com/google/copybara/credentials/TtlSecretTest.java similarity index 100% rename from javatests/com/google/copybara/credentials/TtlSecretTest.java rename to java/javatests/com/google/copybara/credentials/TtlSecretTest.java diff --git a/javatests/com/google/copybara/exception/BUILD b/java/javatests/com/google/copybara/exception/BUILD similarity index 100% rename from javatests/com/google/copybara/exception/BUILD rename to java/javatests/com/google/copybara/exception/BUILD diff --git a/javatests/com/google/copybara/exception/ValidationExceptionTest.java b/java/javatests/com/google/copybara/exception/ValidationExceptionTest.java similarity index 100% rename from javatests/com/google/copybara/exception/ValidationExceptionTest.java rename to java/javatests/com/google/copybara/exception/ValidationExceptionTest.java diff --git a/javatests/com/google/copybara/folder/FolderDestinationReaderTest.java b/java/javatests/com/google/copybara/folder/FolderDestinationReaderTest.java similarity index 100% rename from javatests/com/google/copybara/folder/FolderDestinationReaderTest.java rename to java/javatests/com/google/copybara/folder/FolderDestinationReaderTest.java diff --git a/javatests/com/google/copybara/folder/FolderDestinationTest.java b/java/javatests/com/google/copybara/folder/FolderDestinationTest.java similarity index 100% rename from javatests/com/google/copybara/folder/FolderDestinationTest.java rename to java/javatests/com/google/copybara/folder/FolderDestinationTest.java diff --git a/javatests/com/google/copybara/folder/FolderOriginTest.java b/java/javatests/com/google/copybara/folder/FolderOriginTest.java similarity index 100% rename from javatests/com/google/copybara/folder/FolderOriginTest.java rename to java/javatests/com/google/copybara/folder/FolderOriginTest.java diff --git a/javatests/com/google/copybara/format/BUILD b/java/javatests/com/google/copybara/format/BUILD similarity index 100% rename from javatests/com/google/copybara/format/BUILD rename to java/javatests/com/google/copybara/format/BUILD diff --git a/javatests/com/google/copybara/format/BuildifierFormatTest.java b/java/javatests/com/google/copybara/format/BuildifierFormatTest.java similarity index 100% rename from javatests/com/google/copybara/format/BuildifierFormatTest.java rename to java/javatests/com/google/copybara/format/BuildifierFormatTest.java diff --git a/javatests/com/google/copybara/git/BUILD b/java/javatests/com/google/copybara/git/BUILD similarity index 100% rename from javatests/com/google/copybara/git/BUILD rename to java/javatests/com/google/copybara/git/BUILD diff --git a/javatests/com/google/copybara/git/CredentialFileHandlerTest.java b/java/javatests/com/google/copybara/git/CredentialFileHandlerTest.java similarity index 100% rename from javatests/com/google/copybara/git/CredentialFileHandlerTest.java rename to java/javatests/com/google/copybara/git/CredentialFileHandlerTest.java diff --git a/javatests/com/google/copybara/git/FuzzyClosestVersionSelectorTest.java b/java/javatests/com/google/copybara/git/FuzzyClosestVersionSelectorTest.java similarity index 100% rename from javatests/com/google/copybara/git/FuzzyClosestVersionSelectorTest.java rename to java/javatests/com/google/copybara/git/FuzzyClosestVersionSelectorTest.java diff --git a/javatests/com/google/copybara/git/GerritDestinationTest.java b/java/javatests/com/google/copybara/git/GerritDestinationTest.java similarity index 100% rename from javatests/com/google/copybara/git/GerritDestinationTest.java rename to java/javatests/com/google/copybara/git/GerritDestinationTest.java diff --git a/javatests/com/google/copybara/git/GerritEndpointTest.java b/java/javatests/com/google/copybara/git/GerritEndpointTest.java similarity index 100% rename from javatests/com/google/copybara/git/GerritEndpointTest.java rename to java/javatests/com/google/copybara/git/GerritEndpointTest.java diff --git a/javatests/com/google/copybara/git/GerritOptionsTest.java b/java/javatests/com/google/copybara/git/GerritOptionsTest.java similarity index 100% rename from javatests/com/google/copybara/git/GerritOptionsTest.java rename to java/javatests/com/google/copybara/git/GerritOptionsTest.java diff --git a/javatests/com/google/copybara/git/GerritOriginTest.java b/java/javatests/com/google/copybara/git/GerritOriginTest.java similarity index 100% rename from javatests/com/google/copybara/git/GerritOriginTest.java rename to java/javatests/com/google/copybara/git/GerritOriginTest.java diff --git a/javatests/com/google/copybara/git/GerritTriggerTest.java b/java/javatests/com/google/copybara/git/GerritTriggerTest.java similarity index 100% rename from javatests/com/google/copybara/git/GerritTriggerTest.java rename to java/javatests/com/google/copybara/git/GerritTriggerTest.java diff --git a/javatests/com/google/copybara/git/GitCredentialTest.java b/java/javatests/com/google/copybara/git/GitCredentialTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitCredentialTest.java rename to java/javatests/com/google/copybara/git/GitCredentialTest.java diff --git a/javatests/com/google/copybara/git/GitDestinationIntegrateTest.java b/java/javatests/com/google/copybara/git/GitDestinationIntegrateTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitDestinationIntegrateTest.java rename to java/javatests/com/google/copybara/git/GitDestinationIntegrateTest.java diff --git a/javatests/com/google/copybara/git/GitDestinationReaderTest.java b/java/javatests/com/google/copybara/git/GitDestinationReaderTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitDestinationReaderTest.java rename to java/javatests/com/google/copybara/git/GitDestinationReaderTest.java diff --git a/javatests/com/google/copybara/git/GitDestinationTest.java b/java/javatests/com/google/copybara/git/GitDestinationTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitDestinationTest.java rename to java/javatests/com/google/copybara/git/GitDestinationTest.java diff --git a/javatests/com/google/copybara/git/GitEnvironmentTest.java b/java/javatests/com/google/copybara/git/GitEnvironmentTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitEnvironmentTest.java rename to java/javatests/com/google/copybara/git/GitEnvironmentTest.java diff --git a/javatests/com/google/copybara/git/GitHubDestinationTest.java b/java/javatests/com/google/copybara/git/GitHubDestinationTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubDestinationTest.java rename to java/javatests/com/google/copybara/git/GitHubDestinationTest.java diff --git a/javatests/com/google/copybara/git/GitHubEndpointTest.java b/java/javatests/com/google/copybara/git/GitHubEndpointTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubEndpointTest.java rename to java/javatests/com/google/copybara/git/GitHubEndpointTest.java diff --git a/javatests/com/google/copybara/git/GitHubOptionsTest.java b/java/javatests/com/google/copybara/git/GitHubOptionsTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubOptionsTest.java rename to java/javatests/com/google/copybara/git/GitHubOptionsTest.java diff --git a/javatests/com/google/copybara/git/GitHubPostSubmitApprovalsProviderTest.java b/java/javatests/com/google/copybara/git/GitHubPostSubmitApprovalsProviderTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubPostSubmitApprovalsProviderTest.java rename to java/javatests/com/google/copybara/git/GitHubPostSubmitApprovalsProviderTest.java diff --git a/javatests/com/google/copybara/git/GitHubPrDestinationTest.java b/java/javatests/com/google/copybara/git/GitHubPrDestinationTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubPrDestinationTest.java rename to java/javatests/com/google/copybara/git/GitHubPrDestinationTest.java diff --git a/javatests/com/google/copybara/git/GitHubPrIntegrateLabelTest.java b/java/javatests/com/google/copybara/git/GitHubPrIntegrateLabelTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubPrIntegrateLabelTest.java rename to java/javatests/com/google/copybara/git/GitHubPrIntegrateLabelTest.java diff --git a/javatests/com/google/copybara/git/GitHubPrOriginTest.java b/java/javatests/com/google/copybara/git/GitHubPrOriginTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubPrOriginTest.java rename to java/javatests/com/google/copybara/git/GitHubPrOriginTest.java diff --git a/javatests/com/google/copybara/git/GitHubPreSubmitApprovalsProviderTest.java b/java/javatests/com/google/copybara/git/GitHubPreSubmitApprovalsProviderTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubPreSubmitApprovalsProviderTest.java rename to java/javatests/com/google/copybara/git/GitHubPreSubmitApprovalsProviderTest.java diff --git a/javatests/com/google/copybara/git/GitHubRepositoryHookTest.java b/java/javatests/com/google/copybara/git/GitHubRepositoryHookTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubRepositoryHookTest.java rename to java/javatests/com/google/copybara/git/GitHubRepositoryHookTest.java diff --git a/javatests/com/google/copybara/git/GitHubSecuritySettingsValidatorTest.java b/java/javatests/com/google/copybara/git/GitHubSecuritySettingsValidatorTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubSecuritySettingsValidatorTest.java rename to java/javatests/com/google/copybara/git/GitHubSecuritySettingsValidatorTest.java diff --git a/javatests/com/google/copybara/git/GitHubTriggerTest.java b/java/javatests/com/google/copybara/git/GitHubTriggerTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubTriggerTest.java rename to java/javatests/com/google/copybara/git/GitHubTriggerTest.java diff --git a/javatests/com/google/copybara/git/GitHubUserApprovalsValidatorTest.java b/java/javatests/com/google/copybara/git/GitHubUserApprovalsValidatorTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitHubUserApprovalsValidatorTest.java rename to java/javatests/com/google/copybara/git/GitHubUserApprovalsValidatorTest.java diff --git a/javatests/com/google/copybara/git/GitLabMrDestinationTest.java b/java/javatests/com/google/copybara/git/GitLabMrDestinationTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitLabMrDestinationTest.java rename to java/javatests/com/google/copybara/git/GitLabMrDestinationTest.java diff --git a/javatests/com/google/copybara/git/GitLabMrOriginTest.java b/java/javatests/com/google/copybara/git/GitLabMrOriginTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitLabMrOriginTest.java rename to java/javatests/com/google/copybara/git/GitLabMrOriginTest.java diff --git a/javatests/com/google/copybara/git/GitLabMrWriteHookTest.java b/java/javatests/com/google/copybara/git/GitLabMrWriteHookTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitLabMrWriteHookTest.java rename to java/javatests/com/google/copybara/git/GitLabMrWriteHookTest.java diff --git a/javatests/com/google/copybara/git/GitLabMrWriterTest.java b/java/javatests/com/google/copybara/git/GitLabMrWriterTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitLabMrWriterTest.java rename to java/javatests/com/google/copybara/git/GitLabMrWriterTest.java diff --git a/javatests/com/google/copybara/git/GitMirrorTest.java b/java/javatests/com/google/copybara/git/GitMirrorTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitMirrorTest.java rename to java/javatests/com/google/copybara/git/GitMirrorTest.java diff --git a/javatests/com/google/copybara/git/GitOriginSubmodulesTest.java b/java/javatests/com/google/copybara/git/GitOriginSubmodulesTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitOriginSubmodulesTest.java rename to java/javatests/com/google/copybara/git/GitOriginSubmodulesTest.java diff --git a/javatests/com/google/copybara/git/GitOriginTest.java b/java/javatests/com/google/copybara/git/GitOriginTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitOriginTest.java rename to java/javatests/com/google/copybara/git/GitOriginTest.java diff --git a/javatests/com/google/copybara/git/GitRepoTypeTest.java b/java/javatests/com/google/copybara/git/GitRepoTypeTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitRepoTypeTest.java rename to java/javatests/com/google/copybara/git/GitRepoTypeTest.java diff --git a/javatests/com/google/copybara/git/GitRepositoryTest.java b/java/javatests/com/google/copybara/git/GitRepositoryTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitRepositoryTest.java rename to java/javatests/com/google/copybara/git/GitRepositoryTest.java diff --git a/javatests/com/google/copybara/git/GitRevisionTest.java b/java/javatests/com/google/copybara/git/GitRevisionTest.java similarity index 100% rename from javatests/com/google/copybara/git/GitRevisionTest.java rename to java/javatests/com/google/copybara/git/GitRevisionTest.java diff --git a/javatests/com/google/copybara/git/LatestVersionSelectorForGitTest.java b/java/javatests/com/google/copybara/git/LatestVersionSelectorForGitTest.java similarity index 100% rename from javatests/com/google/copybara/git/LatestVersionSelectorForGitTest.java rename to java/javatests/com/google/copybara/git/LatestVersionSelectorForGitTest.java diff --git a/javatests/com/google/copybara/git/RefspecTest.java b/java/javatests/com/google/copybara/git/RefspecTest.java similarity index 100% rename from javatests/com/google/copybara/git/RefspecTest.java rename to java/javatests/com/google/copybara/git/RefspecTest.java diff --git a/javatests/com/google/copybara/git/SameGitTreeTest.java b/java/javatests/com/google/copybara/git/SameGitTreeTest.java similarity index 100% rename from javatests/com/google/copybara/git/SameGitTreeTest.java rename to java/javatests/com/google/copybara/git/SameGitTreeTest.java diff --git a/javatests/com/google/copybara/git/SubmodulesInDestinationTest.java b/java/javatests/com/google/copybara/git/SubmodulesInDestinationTest.java similarity index 100% rename from javatests/com/google/copybara/git/SubmodulesInDestinationTest.java rename to java/javatests/com/google/copybara/git/SubmodulesInDestinationTest.java diff --git a/javatests/com/google/copybara/git/gerritapi/GerritApiTest.java b/java/javatests/com/google/copybara/git/gerritapi/GerritApiTest.java similarity index 100% rename from javatests/com/google/copybara/git/gerritapi/GerritApiTest.java rename to java/javatests/com/google/copybara/git/gerritapi/GerritApiTest.java diff --git a/javatests/com/google/copybara/git/gerritapi/GerritApiTransportWithCheckerTest.java b/java/javatests/com/google/copybara/git/gerritapi/GerritApiTransportWithCheckerTest.java similarity index 100% rename from javatests/com/google/copybara/git/gerritapi/GerritApiTransportWithCheckerTest.java rename to java/javatests/com/google/copybara/git/gerritapi/GerritApiTransportWithCheckerTest.java diff --git a/javatests/com/google/copybara/git/github/api/GitHubApiTest.java b/java/javatests/com/google/copybara/git/github/api/GitHubApiTest.java similarity index 100% rename from javatests/com/google/copybara/git/github/api/GitHubApiTest.java rename to java/javatests/com/google/copybara/git/github/api/GitHubApiTest.java diff --git a/javatests/com/google/copybara/git/github/api/GitHubApiTransportImplTest.java b/java/javatests/com/google/copybara/git/github/api/GitHubApiTransportImplTest.java similarity index 100% rename from javatests/com/google/copybara/git/github/api/GitHubApiTransportImplTest.java rename to java/javatests/com/google/copybara/git/github/api/GitHubApiTransportImplTest.java diff --git a/javatests/com/google/copybara/git/github/api/GitHubApiTransportWithCheckerTest.java b/java/javatests/com/google/copybara/git/github/api/GitHubApiTransportWithCheckerTest.java similarity index 100% rename from javatests/com/google/copybara/git/github/api/GitHubApiTransportWithCheckerTest.java rename to java/javatests/com/google/copybara/git/github/api/GitHubApiTransportWithCheckerTest.java diff --git a/javatests/com/google/copybara/git/github/api/GitHubGraphQLTest.java b/java/javatests/com/google/copybara/git/github/api/GitHubGraphQLTest.java similarity index 100% rename from javatests/com/google/copybara/git/github/api/GitHubGraphQLTest.java rename to java/javatests/com/google/copybara/git/github/api/GitHubGraphQLTest.java diff --git a/javatests/com/google/copybara/git/github/util/BUILD b/java/javatests/com/google/copybara/git/github/util/BUILD similarity index 100% rename from javatests/com/google/copybara/git/github/util/BUILD rename to java/javatests/com/google/copybara/git/github/util/BUILD diff --git a/javatests/com/google/copybara/git/github/util/GitHubHostTest.java b/java/javatests/com/google/copybara/git/github/util/GitHubHostTest.java similarity index 100% rename from javatests/com/google/copybara/git/github/util/GitHubHostTest.java rename to java/javatests/com/google/copybara/git/github/util/GitHubHostTest.java diff --git a/javatests/com/google/copybara/git/github/util/GitHubIdentifierTest.java b/java/javatests/com/google/copybara/git/github/util/GitHubIdentifierTest.java similarity index 100% rename from javatests/com/google/copybara/git/github/util/GitHubIdentifierTest.java rename to java/javatests/com/google/copybara/git/github/util/GitHubIdentifierTest.java diff --git a/javatests/com/google/copybara/git/github/util/GitHubUtilTest.java b/java/javatests/com/google/copybara/git/github/util/GitHubUtilTest.java similarity index 100% rename from javatests/com/google/copybara/git/github/util/GitHubUtilTest.java rename to java/javatests/com/google/copybara/git/github/util/GitHubUtilTest.java diff --git a/javatests/com/google/copybara/git/gitlab/BUILD b/java/javatests/com/google/copybara/git/gitlab/BUILD similarity index 100% rename from javatests/com/google/copybara/git/gitlab/BUILD rename to java/javatests/com/google/copybara/git/gitlab/BUILD diff --git a/javatests/com/google/copybara/git/gitlab/GitLabOptionsTest.java b/java/javatests/com/google/copybara/git/gitlab/GitLabOptionsTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/GitLabOptionsTest.java rename to java/javatests/com/google/copybara/git/gitlab/GitLabOptionsTest.java diff --git a/javatests/com/google/copybara/git/gitlab/GitLabUtilTest.java b/java/javatests/com/google/copybara/git/gitlab/GitLabUtilTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/GitLabUtilTest.java rename to java/javatests/com/google/copybara/git/gitlab/GitLabUtilTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/BUILD b/java/javatests/com/google/copybara/git/gitlab/api/BUILD similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/BUILD rename to java/javatests/com/google/copybara/git/gitlab/api/BUILD diff --git a/javatests/com/google/copybara/git/gitlab/api/GitLabApiTest.java b/java/javatests/com/google/copybara/git/gitlab/api/GitLabApiTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/GitLabApiTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/GitLabApiTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/GitLabApiTransportImplTest.java b/java/javatests/com/google/copybara/git/gitlab/api/GitLabApiTransportImplTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/GitLabApiTransportImplTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/GitLabApiTransportImplTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/BUILD b/java/javatests/com/google/copybara/git/gitlab/api/entities/BUILD similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/BUILD rename to java/javatests/com/google/copybara/git/gitlab/api/entities/BUILD diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/CreateMergeRequestParamsTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/CreateMergeRequestParamsTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/CreateMergeRequestParamsTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/CreateMergeRequestParamsTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/ExternalStatusCheckTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/ExternalStatusCheckTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/ExternalStatusCheckTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/ExternalStatusCheckTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/GitLabApiParamsTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/GitLabApiParamsTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/GitLabApiParamsTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/GitLabApiParamsTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/ListProjectMergeRequestParamsTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/ListProjectMergeRequestParamsTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/ListProjectMergeRequestParamsTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/ListProjectMergeRequestParamsTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/ListUsersParamsTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/ListUsersParamsTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/ListUsersParamsTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/ListUsersParamsTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/MergeRequestTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/MergeRequestTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/MergeRequestTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/MergeRequestTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/PaginatedPageListTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/PaginatedPageListTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/PaginatedPageListTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/PaginatedPageListTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/ProjectTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/ProjectTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/ProjectTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/ProjectTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/SetExternalStatusCheckParamsTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/SetExternalStatusCheckParamsTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/SetExternalStatusCheckParamsTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/SetExternalStatusCheckParamsTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/SetExternalStatusCheckResponseTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/SetExternalStatusCheckResponseTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/SetExternalStatusCheckResponseTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/SetExternalStatusCheckResponseTest.java diff --git a/javatests/com/google/copybara/git/gitlab/api/entities/UpdateMergeRequestParamsTest.java b/java/javatests/com/google/copybara/git/gitlab/api/entities/UpdateMergeRequestParamsTest.java similarity index 100% rename from javatests/com/google/copybara/git/gitlab/api/entities/UpdateMergeRequestParamsTest.java rename to java/javatests/com/google/copybara/git/gitlab/api/entities/UpdateMergeRequestParamsTest.java diff --git a/javatests/com/google/copybara/go/BUILD b/java/javatests/com/google/copybara/go/BUILD similarity index 100% rename from javatests/com/google/copybara/go/BUILD rename to java/javatests/com/google/copybara/go/BUILD diff --git a/javatests/com/google/copybara/go/GoProxyVersioningTest.java b/java/javatests/com/google/copybara/go/GoProxyVersioningTest.java similarity index 100% rename from javatests/com/google/copybara/go/GoProxyVersioningTest.java rename to java/javatests/com/google/copybara/go/GoProxyVersioningTest.java diff --git a/javatests/com/google/copybara/go/PseudoVersionSelectorTest.java b/java/javatests/com/google/copybara/go/PseudoVersionSelectorTest.java similarity index 100% rename from javatests/com/google/copybara/go/PseudoVersionSelectorTest.java rename to java/javatests/com/google/copybara/go/PseudoVersionSelectorTest.java diff --git a/javatests/com/google/copybara/hashing/BUILD b/java/javatests/com/google/copybara/hashing/BUILD similarity index 100% rename from javatests/com/google/copybara/hashing/BUILD rename to java/javatests/com/google/copybara/hashing/BUILD diff --git a/javatests/com/google/copybara/hashing/HashingModuleTest.java b/java/javatests/com/google/copybara/hashing/HashingModuleTest.java similarity index 100% rename from javatests/com/google/copybara/hashing/HashingModuleTest.java rename to java/javatests/com/google/copybara/hashing/HashingModuleTest.java diff --git a/javatests/com/google/copybara/hg/BUILD b/java/javatests/com/google/copybara/hg/BUILD similarity index 100% rename from javatests/com/google/copybara/hg/BUILD rename to java/javatests/com/google/copybara/hg/BUILD diff --git a/javatests/com/google/copybara/hg/HgDestinationTest.java b/java/javatests/com/google/copybara/hg/HgDestinationTest.java similarity index 100% rename from javatests/com/google/copybara/hg/HgDestinationTest.java rename to java/javatests/com/google/copybara/hg/HgDestinationTest.java diff --git a/javatests/com/google/copybara/hg/HgOriginTest.java b/java/javatests/com/google/copybara/hg/HgOriginTest.java similarity index 100% rename from javatests/com/google/copybara/hg/HgOriginTest.java rename to java/javatests/com/google/copybara/hg/HgOriginTest.java diff --git a/javatests/com/google/copybara/hg/HgRepositoryTest.java b/java/javatests/com/google/copybara/hg/HgRepositoryTest.java similarity index 100% rename from javatests/com/google/copybara/hg/HgRepositoryTest.java rename to java/javatests/com/google/copybara/hg/HgRepositoryTest.java diff --git a/javatests/com/google/copybara/html/BUILD b/java/javatests/com/google/copybara/html/BUILD similarity index 100% rename from javatests/com/google/copybara/html/BUILD rename to java/javatests/com/google/copybara/html/BUILD diff --git a/javatests/com/google/copybara/html/HtmlTest.java b/java/javatests/com/google/copybara/html/HtmlTest.java similarity index 100% rename from javatests/com/google/copybara/html/HtmlTest.java rename to java/javatests/com/google/copybara/html/HtmlTest.java diff --git a/javatests/com/google/copybara/http/BUILD b/java/javatests/com/google/copybara/http/BUILD similarity index 100% rename from javatests/com/google/copybara/http/BUILD rename to java/javatests/com/google/copybara/http/BUILD diff --git a/javatests/com/google/copybara/http/HttpModuleTest.java b/java/javatests/com/google/copybara/http/HttpModuleTest.java similarity index 100% rename from javatests/com/google/copybara/http/HttpModuleTest.java rename to java/javatests/com/google/copybara/http/HttpModuleTest.java diff --git a/javatests/com/google/copybara/http/auth/AuthTest.java b/java/javatests/com/google/copybara/http/auth/AuthTest.java similarity index 100% rename from javatests/com/google/copybara/http/auth/AuthTest.java rename to java/javatests/com/google/copybara/http/auth/AuthTest.java diff --git a/javatests/com/google/copybara/http/auth/BUILD b/java/javatests/com/google/copybara/http/auth/BUILD similarity index 100% rename from javatests/com/google/copybara/http/auth/BUILD rename to java/javatests/com/google/copybara/http/auth/BUILD diff --git a/javatests/com/google/copybara/http/endpoint/BUILD b/java/javatests/com/google/copybara/http/endpoint/BUILD similarity index 100% rename from javatests/com/google/copybara/http/endpoint/BUILD rename to java/javatests/com/google/copybara/http/endpoint/BUILD diff --git a/javatests/com/google/copybara/http/endpoint/HttpEndpointTest.java b/java/javatests/com/google/copybara/http/endpoint/HttpEndpointTest.java similarity index 100% rename from javatests/com/google/copybara/http/endpoint/HttpEndpointTest.java rename to java/javatests/com/google/copybara/http/endpoint/HttpEndpointTest.java diff --git a/javatests/com/google/copybara/http/json/BUILD b/java/javatests/com/google/copybara/http/json/BUILD similarity index 100% rename from javatests/com/google/copybara/http/json/BUILD rename to java/javatests/com/google/copybara/http/json/BUILD diff --git a/javatests/com/google/copybara/http/json/HttpEndpointJsonContentTest.java b/java/javatests/com/google/copybara/http/json/HttpEndpointJsonContentTest.java similarity index 100% rename from javatests/com/google/copybara/http/json/HttpEndpointJsonContentTest.java rename to java/javatests/com/google/copybara/http/json/HttpEndpointJsonContentTest.java diff --git a/javatests/com/google/copybara/http/multipart/BUILD b/java/javatests/com/google/copybara/http/multipart/BUILD similarity index 100% rename from javatests/com/google/copybara/http/multipart/BUILD rename to java/javatests/com/google/copybara/http/multipart/BUILD diff --git a/javatests/com/google/copybara/http/multipart/HttpEndpointMultipartContentTest.java b/java/javatests/com/google/copybara/http/multipart/HttpEndpointMultipartContentTest.java similarity index 100% rename from javatests/com/google/copybara/http/multipart/HttpEndpointMultipartContentTest.java rename to java/javatests/com/google/copybara/http/multipart/HttpEndpointMultipartContentTest.java diff --git a/javatests/com/google/copybara/http/multipart/HttpEndpointUrlEncodedFormContentTest.java b/java/javatests/com/google/copybara/http/multipart/HttpEndpointUrlEncodedFormContentTest.java similarity index 100% rename from javatests/com/google/copybara/http/multipart/HttpEndpointUrlEncodedFormContentTest.java rename to java/javatests/com/google/copybara/http/multipart/HttpEndpointUrlEncodedFormContentTest.java diff --git a/javatests/com/google/copybara/jcommander/BUILD b/java/javatests/com/google/copybara/jcommander/BUILD similarity index 100% rename from javatests/com/google/copybara/jcommander/BUILD rename to java/javatests/com/google/copybara/jcommander/BUILD diff --git a/javatests/com/google/copybara/jcommander/DurationConverterTest.java b/java/javatests/com/google/copybara/jcommander/DurationConverterTest.java similarity index 100% rename from javatests/com/google/copybara/jcommander/DurationConverterTest.java rename to java/javatests/com/google/copybara/jcommander/DurationConverterTest.java diff --git a/javatests/com/google/copybara/jcommander/GlobConverterTest.java b/java/javatests/com/google/copybara/jcommander/GlobConverterTest.java similarity index 100% rename from javatests/com/google/copybara/jcommander/GlobConverterTest.java rename to java/javatests/com/google/copybara/jcommander/GlobConverterTest.java diff --git a/javatests/com/google/copybara/jcommander/SemicolonListConverterTest.java b/java/javatests/com/google/copybara/jcommander/SemicolonListConverterTest.java similarity index 100% rename from javatests/com/google/copybara/jcommander/SemicolonListConverterTest.java rename to java/javatests/com/google/copybara/jcommander/SemicolonListConverterTest.java diff --git a/javatests/com/google/copybara/json/BUILD b/java/javatests/com/google/copybara/json/BUILD similarity index 100% rename from javatests/com/google/copybara/json/BUILD rename to java/javatests/com/google/copybara/json/BUILD diff --git a/javatests/com/google/copybara/json/GsonParserUtilTest.java b/java/javatests/com/google/copybara/json/GsonParserUtilTest.java similarity index 100% rename from javatests/com/google/copybara/json/GsonParserUtilTest.java rename to java/javatests/com/google/copybara/json/GsonParserUtilTest.java diff --git a/javatests/com/google/copybara/monitor/BUILD b/java/javatests/com/google/copybara/monitor/BUILD similarity index 100% rename from javatests/com/google/copybara/monitor/BUILD rename to java/javatests/com/google/copybara/monitor/BUILD diff --git a/javatests/com/google/copybara/monitor/ConsoleEventMonitorTest.java b/java/javatests/com/google/copybara/monitor/ConsoleEventMonitorTest.java similarity index 100% rename from javatests/com/google/copybara/monitor/ConsoleEventMonitorTest.java rename to java/javatests/com/google/copybara/monitor/ConsoleEventMonitorTest.java diff --git a/javatests/com/google/copybara/onboard/BUILD b/java/javatests/com/google/copybara/onboard/BUILD similarity index 100% rename from javatests/com/google/copybara/onboard/BUILD rename to java/javatests/com/google/copybara/onboard/BUILD diff --git a/javatests/com/google/copybara/onboard/CommandLineGuideTest.java b/java/javatests/com/google/copybara/onboard/CommandLineGuideTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/CommandLineGuideTest.java rename to java/javatests/com/google/copybara/onboard/CommandLineGuideTest.java diff --git a/javatests/com/google/copybara/onboard/ConfigBuilderTest.java b/java/javatests/com/google/copybara/onboard/ConfigBuilderTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/ConfigBuilderTest.java rename to java/javatests/com/google/copybara/onboard/ConfigBuilderTest.java diff --git a/javatests/com/google/copybara/onboard/ConfigHeuristicsInputProviderTest.java b/java/javatests/com/google/copybara/onboard/ConfigHeuristicsInputProviderTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/ConfigHeuristicsInputProviderTest.java rename to java/javatests/com/google/copybara/onboard/ConfigHeuristicsInputProviderTest.java diff --git a/javatests/com/google/copybara/onboard/GeneratorCmdTest.java b/java/javatests/com/google/copybara/onboard/GeneratorCmdTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/GeneratorCmdTest.java rename to java/javatests/com/google/copybara/onboard/GeneratorCmdTest.java diff --git a/javatests/com/google/copybara/onboard/GitToGitGeneratorTest.java b/java/javatests/com/google/copybara/onboard/GitToGitGeneratorTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/GitToGitGeneratorTest.java rename to java/javatests/com/google/copybara/onboard/GitToGitGeneratorTest.java diff --git a/javatests/com/google/copybara/onboard/InputsTest.java b/java/javatests/com/google/copybara/onboard/InputsTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/InputsTest.java rename to java/javatests/com/google/copybara/onboard/InputsTest.java diff --git a/javatests/com/google/copybara/onboard/OnboardCmdTest.java b/java/javatests/com/google/copybara/onboard/OnboardCmdTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/OnboardCmdTest.java rename to java/javatests/com/google/copybara/onboard/OnboardCmdTest.java diff --git a/javatests/com/google/copybara/onboard/StarlarkConverterTest.java b/java/javatests/com/google/copybara/onboard/StarlarkConverterTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/StarlarkConverterTest.java rename to java/javatests/com/google/copybara/onboard/StarlarkConverterTest.java diff --git a/javatests/com/google/copybara/onboard/core/AskInputProviderTest.java b/java/javatests/com/google/copybara/onboard/core/AskInputProviderTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/core/AskInputProviderTest.java rename to java/javatests/com/google/copybara/onboard/core/AskInputProviderTest.java diff --git a/javatests/com/google/copybara/onboard/core/BUILD b/java/javatests/com/google/copybara/onboard/core/BUILD similarity index 100% rename from javatests/com/google/copybara/onboard/core/BUILD rename to java/javatests/com/google/copybara/onboard/core/BUILD diff --git a/javatests/com/google/copybara/onboard/core/CachedInputProviderTest.java b/java/javatests/com/google/copybara/onboard/core/CachedInputProviderTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/core/CachedInputProviderTest.java rename to java/javatests/com/google/copybara/onboard/core/CachedInputProviderTest.java diff --git a/javatests/com/google/copybara/onboard/core/ConstantProviderTest.java b/java/javatests/com/google/copybara/onboard/core/ConstantProviderTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/core/ConstantProviderTest.java rename to java/javatests/com/google/copybara/onboard/core/ConstantProviderTest.java diff --git a/javatests/com/google/copybara/onboard/core/InputProviderResolverImplTest.java b/java/javatests/com/google/copybara/onboard/core/InputProviderResolverImplTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/core/InputProviderResolverImplTest.java rename to java/javatests/com/google/copybara/onboard/core/InputProviderResolverImplTest.java diff --git a/javatests/com/google/copybara/onboard/core/InputTest.java b/java/javatests/com/google/copybara/onboard/core/InputTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/core/InputTest.java rename to java/javatests/com/google/copybara/onboard/core/InputTest.java diff --git a/javatests/com/google/copybara/onboard/core/MapBasedInputProviderTest.java b/java/javatests/com/google/copybara/onboard/core/MapBasedInputProviderTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/core/MapBasedInputProviderTest.java rename to java/javatests/com/google/copybara/onboard/core/MapBasedInputProviderTest.java diff --git a/javatests/com/google/copybara/onboard/core/PrioritizedInputProviderTest.java b/java/javatests/com/google/copybara/onboard/core/PrioritizedInputProviderTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/core/PrioritizedInputProviderTest.java rename to java/javatests/com/google/copybara/onboard/core/PrioritizedInputProviderTest.java diff --git a/javatests/com/google/copybara/onboard/core/template/TemplateConfigGeneratorTest.java b/java/javatests/com/google/copybara/onboard/core/template/TemplateConfigGeneratorTest.java similarity index 100% rename from javatests/com/google/copybara/onboard/core/template/TemplateConfigGeneratorTest.java rename to java/javatests/com/google/copybara/onboard/core/template/TemplateConfigGeneratorTest.java diff --git a/javatests/com/google/copybara/profiler/BUILD b/java/javatests/com/google/copybara/profiler/BUILD similarity index 100% rename from javatests/com/google/copybara/profiler/BUILD rename to java/javatests/com/google/copybara/profiler/BUILD diff --git a/javatests/com/google/copybara/profiler/ConsoleProfilerListenerTest.java b/java/javatests/com/google/copybara/profiler/ConsoleProfilerListenerTest.java similarity index 100% rename from javatests/com/google/copybara/profiler/ConsoleProfilerListenerTest.java rename to java/javatests/com/google/copybara/profiler/ConsoleProfilerListenerTest.java diff --git a/javatests/com/google/copybara/profiler/ProfilerTest.java b/java/javatests/com/google/copybara/profiler/ProfilerTest.java similarity index 100% rename from javatests/com/google/copybara/profiler/ProfilerTest.java rename to java/javatests/com/google/copybara/profiler/ProfilerTest.java diff --git a/javatests/com/google/copybara/python/BUILD b/java/javatests/com/google/copybara/python/BUILD similarity index 100% rename from javatests/com/google/copybara/python/BUILD rename to java/javatests/com/google/copybara/python/BUILD diff --git a/javatests/com/google/copybara/python/PackageMetadataTest.java b/java/javatests/com/google/copybara/python/PackageMetadataTest.java similarity index 100% rename from javatests/com/google/copybara/python/PackageMetadataTest.java rename to java/javatests/com/google/copybara/python/PackageMetadataTest.java diff --git a/javatests/com/google/copybara/python/testfiles/TESTMETADATA b/java/javatests/com/google/copybara/python/testfiles/TESTMETADATA similarity index 100% rename from javatests/com/google/copybara/python/testfiles/TESTMETADATA rename to java/javatests/com/google/copybara/python/testfiles/TESTMETADATA diff --git a/javatests/com/google/copybara/re2/BUILD b/java/javatests/com/google/copybara/re2/BUILD similarity index 100% rename from javatests/com/google/copybara/re2/BUILD rename to java/javatests/com/google/copybara/re2/BUILD diff --git a/javatests/com/google/copybara/re2/Re2Test.java b/java/javatests/com/google/copybara/re2/Re2Test.java similarity index 100% rename from javatests/com/google/copybara/re2/Re2Test.java rename to java/javatests/com/google/copybara/re2/Re2Test.java diff --git a/javatests/com/google/copybara/regenerate/BUILD b/java/javatests/com/google/copybara/regenerate/BUILD similarity index 100% rename from javatests/com/google/copybara/regenerate/BUILD rename to java/javatests/com/google/copybara/regenerate/BUILD diff --git a/javatests/com/google/copybara/regenerate/RegenerateCmdTest.java b/java/javatests/com/google/copybara/regenerate/RegenerateCmdTest.java similarity index 100% rename from javatests/com/google/copybara/regenerate/RegenerateCmdTest.java rename to java/javatests/com/google/copybara/regenerate/RegenerateCmdTest.java diff --git a/javatests/com/google/copybara/remotefile/BUILD b/java/javatests/com/google/copybara/remotefile/BUILD similarity index 100% rename from javatests/com/google/copybara/remotefile/BUILD rename to java/javatests/com/google/copybara/remotefile/BUILD diff --git a/javatests/com/google/copybara/remotefile/GithubArchiveTest.java b/java/javatests/com/google/copybara/remotefile/GithubArchiveTest.java similarity index 100% rename from javatests/com/google/copybara/remotefile/GithubArchiveTest.java rename to java/javatests/com/google/copybara/remotefile/GithubArchiveTest.java diff --git a/javatests/com/google/copybara/remotefile/RemoteArchiveOriginTest.java b/java/javatests/com/google/copybara/remotefile/RemoteArchiveOriginTest.java similarity index 100% rename from javatests/com/google/copybara/remotefile/RemoteArchiveOriginTest.java rename to java/javatests/com/google/copybara/remotefile/RemoteArchiveOriginTest.java diff --git a/javatests/com/google/copybara/remotefile/RemoteFileModuleTest.java b/java/javatests/com/google/copybara/remotefile/RemoteFileModuleTest.java similarity index 100% rename from javatests/com/google/copybara/remotefile/RemoteFileModuleTest.java rename to java/javatests/com/google/copybara/remotefile/RemoteFileModuleTest.java diff --git a/javatests/com/google/copybara/remotefile/extractutil/BUILD b/java/javatests/com/google/copybara/remotefile/extractutil/BUILD similarity index 100% rename from javatests/com/google/copybara/remotefile/extractutil/BUILD rename to java/javatests/com/google/copybara/remotefile/extractutil/BUILD diff --git a/javatests/com/google/copybara/remotefile/extractutil/ExtractUtilTest.java b/java/javatests/com/google/copybara/remotefile/extractutil/ExtractUtilTest.java similarity index 100% rename from javatests/com/google/copybara/remotefile/extractutil/ExtractUtilTest.java rename to java/javatests/com/google/copybara/remotefile/extractutil/ExtractUtilTest.java diff --git a/javatests/com/google/copybara/rust/BUILD b/java/javatests/com/google/copybara/rust/BUILD similarity index 100% rename from javatests/com/google/copybara/rust/BUILD rename to java/javatests/com/google/copybara/rust/BUILD diff --git a/javatests/com/google/copybara/rust/ComparisonRustVersionRequirementTest.java b/java/javatests/com/google/copybara/rust/ComparisonRustVersionRequirementTest.java similarity index 100% rename from javatests/com/google/copybara/rust/ComparisonRustVersionRequirementTest.java rename to java/javatests/com/google/copybara/rust/ComparisonRustVersionRequirementTest.java diff --git a/javatests/com/google/copybara/rust/DefaultRustVersionRequirementTest.java b/java/javatests/com/google/copybara/rust/DefaultRustVersionRequirementTest.java similarity index 100% rename from javatests/com/google/copybara/rust/DefaultRustVersionRequirementTest.java rename to java/javatests/com/google/copybara/rust/DefaultRustVersionRequirementTest.java diff --git a/javatests/com/google/copybara/rust/EpochRustVersionRequirementTest.java b/java/javatests/com/google/copybara/rust/EpochRustVersionRequirementTest.java similarity index 100% rename from javatests/com/google/copybara/rust/EpochRustVersionRequirementTest.java rename to java/javatests/com/google/copybara/rust/EpochRustVersionRequirementTest.java diff --git a/javatests/com/google/copybara/rust/MultipleRustVersionRequirementTest.java b/java/javatests/com/google/copybara/rust/MultipleRustVersionRequirementTest.java similarity index 100% rename from javatests/com/google/copybara/rust/MultipleRustVersionRequirementTest.java rename to java/javatests/com/google/copybara/rust/MultipleRustVersionRequirementTest.java diff --git a/javatests/com/google/copybara/rust/RustCratesIoVersionListTest.java b/java/javatests/com/google/copybara/rust/RustCratesIoVersionListTest.java similarity index 100% rename from javatests/com/google/copybara/rust/RustCratesIoVersionListTest.java rename to java/javatests/com/google/copybara/rust/RustCratesIoVersionListTest.java diff --git a/javatests/com/google/copybara/rust/RustCratesIoVersionResolverTest.java b/java/javatests/com/google/copybara/rust/RustCratesIoVersionResolverTest.java similarity index 100% rename from javatests/com/google/copybara/rust/RustCratesIoVersionResolverTest.java rename to java/javatests/com/google/copybara/rust/RustCratesIoVersionResolverTest.java diff --git a/javatests/com/google/copybara/rust/RustCratesIoVersionSelectorTest.java b/java/javatests/com/google/copybara/rust/RustCratesIoVersionSelectorTest.java similarity index 100% rename from javatests/com/google/copybara/rust/RustCratesIoVersionSelectorTest.java rename to java/javatests/com/google/copybara/rust/RustCratesIoVersionSelectorTest.java diff --git a/javatests/com/google/copybara/rust/RustModuleTest.java b/java/javatests/com/google/copybara/rust/RustModuleTest.java similarity index 100% rename from javatests/com/google/copybara/rust/RustModuleTest.java rename to java/javatests/com/google/copybara/rust/RustModuleTest.java diff --git a/javatests/com/google/copybara/rust/RustVersionRequirementTest.java b/java/javatests/com/google/copybara/rust/RustVersionRequirementTest.java similarity index 100% rename from javatests/com/google/copybara/rust/RustVersionRequirementTest.java rename to java/javatests/com/google/copybara/rust/RustVersionRequirementTest.java diff --git a/javatests/com/google/copybara/rust/TildeRustVersionRequirementTest.java b/java/javatests/com/google/copybara/rust/TildeRustVersionRequirementTest.java similarity index 100% rename from javatests/com/google/copybara/rust/TildeRustVersionRequirementTest.java rename to java/javatests/com/google/copybara/rust/TildeRustVersionRequirementTest.java diff --git a/javatests/com/google/copybara/rust/WildcardRustVersionRequirementTest.java b/java/javatests/com/google/copybara/rust/WildcardRustVersionRequirementTest.java similarity index 100% rename from javatests/com/google/copybara/rust/WildcardRustVersionRequirementTest.java rename to java/javatests/com/google/copybara/rust/WildcardRustVersionRequirementTest.java diff --git a/javatests/com/google/copybara/templatetoken/BUILD b/java/javatests/com/google/copybara/templatetoken/BUILD similarity index 100% rename from javatests/com/google/copybara/templatetoken/BUILD rename to java/javatests/com/google/copybara/templatetoken/BUILD diff --git a/javatests/com/google/copybara/templatetoken/LabelTemplateTest.java b/java/javatests/com/google/copybara/templatetoken/LabelTemplateTest.java similarity index 100% rename from javatests/com/google/copybara/templatetoken/LabelTemplateTest.java rename to java/javatests/com/google/copybara/templatetoken/LabelTemplateTest.java diff --git a/javatests/com/google/copybara/test.bzl b/java/javatests/com/google/copybara/test.bzl similarity index 100% rename from javatests/com/google/copybara/test.bzl rename to java/javatests/com/google/copybara/test.bzl diff --git a/javatests/com/google/copybara/testing/DummyCheckerTest.java b/java/javatests/com/google/copybara/testing/DummyCheckerTest.java similarity index 100% rename from javatests/com/google/copybara/testing/DummyCheckerTest.java rename to java/javatests/com/google/copybara/testing/DummyCheckerTest.java diff --git a/javatests/com/google/copybara/testing/DummyOriginTest.java b/java/javatests/com/google/copybara/testing/DummyOriginTest.java similarity index 100% rename from javatests/com/google/copybara/testing/DummyOriginTest.java rename to java/javatests/com/google/copybara/testing/DummyOriginTest.java diff --git a/javatests/com/google/copybara/testing/SkylarkTestExecutorTest.java b/java/javatests/com/google/copybara/testing/SkylarkTestExecutorTest.java similarity index 100% rename from javatests/com/google/copybara/testing/SkylarkTestExecutorTest.java rename to java/javatests/com/google/copybara/testing/SkylarkTestExecutorTest.java diff --git a/javatests/com/google/copybara/toml/BUILD b/java/javatests/com/google/copybara/toml/BUILD similarity index 100% rename from javatests/com/google/copybara/toml/BUILD rename to java/javatests/com/google/copybara/toml/BUILD diff --git a/javatests/com/google/copybara/toml/TomlTest.java b/java/javatests/com/google/copybara/toml/TomlTest.java similarity index 100% rename from javatests/com/google/copybara/toml/TomlTest.java rename to java/javatests/com/google/copybara/toml/TomlTest.java diff --git a/javatests/com/google/copybara/transform/BUILD b/java/javatests/com/google/copybara/transform/BUILD similarity index 100% rename from javatests/com/google/copybara/transform/BUILD rename to java/javatests/com/google/copybara/transform/BUILD diff --git a/javatests/com/google/copybara/transform/ConvertEncodingTest.java b/java/javatests/com/google/copybara/transform/ConvertEncodingTest.java similarity index 100% rename from javatests/com/google/copybara/transform/ConvertEncodingTest.java rename to java/javatests/com/google/copybara/transform/ConvertEncodingTest.java diff --git a/javatests/com/google/copybara/transform/CopyOrMoveTest.java b/java/javatests/com/google/copybara/transform/CopyOrMoveTest.java similarity index 100% rename from javatests/com/google/copybara/transform/CopyOrMoveTest.java rename to java/javatests/com/google/copybara/transform/CopyOrMoveTest.java diff --git a/javatests/com/google/copybara/transform/CoreTest.java b/java/javatests/com/google/copybara/transform/CoreTest.java similarity index 100% rename from javatests/com/google/copybara/transform/CoreTest.java rename to java/javatests/com/google/copybara/transform/CoreTest.java diff --git a/javatests/com/google/copybara/transform/ExplicitReversalTest.java b/java/javatests/com/google/copybara/transform/ExplicitReversalTest.java similarity index 100% rename from javatests/com/google/copybara/transform/ExplicitReversalTest.java rename to java/javatests/com/google/copybara/transform/ExplicitReversalTest.java diff --git a/javatests/com/google/copybara/transform/FilterReplaceTest.java b/java/javatests/com/google/copybara/transform/FilterReplaceTest.java similarity index 100% rename from javatests/com/google/copybara/transform/FilterReplaceTest.java rename to java/javatests/com/google/copybara/transform/FilterReplaceTest.java diff --git a/javatests/com/google/copybara/transform/RemoveTest.java b/java/javatests/com/google/copybara/transform/RemoveTest.java similarity index 100% rename from javatests/com/google/copybara/transform/RemoveTest.java rename to java/javatests/com/google/copybara/transform/RemoveTest.java diff --git a/javatests/com/google/copybara/transform/RenameTest.java b/java/javatests/com/google/copybara/transform/RenameTest.java similarity index 100% rename from javatests/com/google/copybara/transform/RenameTest.java rename to java/javatests/com/google/copybara/transform/RenameTest.java diff --git a/javatests/com/google/copybara/transform/ReplaceTest.java b/java/javatests/com/google/copybara/transform/ReplaceTest.java similarity index 100% rename from javatests/com/google/copybara/transform/ReplaceTest.java rename to java/javatests/com/google/copybara/transform/ReplaceTest.java diff --git a/javatests/com/google/copybara/transform/SequenceTest.java b/java/javatests/com/google/copybara/transform/SequenceTest.java similarity index 100% rename from javatests/com/google/copybara/transform/SequenceTest.java rename to java/javatests/com/google/copybara/transform/SequenceTest.java diff --git a/javatests/com/google/copybara/transform/SkylarkConsoleTest.java b/java/javatests/com/google/copybara/transform/SkylarkConsoleTest.java similarity index 100% rename from javatests/com/google/copybara/transform/SkylarkConsoleTest.java rename to java/javatests/com/google/copybara/transform/SkylarkConsoleTest.java diff --git a/javatests/com/google/copybara/transform/SkylarkTransformationTest.java b/java/javatests/com/google/copybara/transform/SkylarkTransformationTest.java similarity index 100% rename from javatests/com/google/copybara/transform/SkylarkTransformationTest.java rename to java/javatests/com/google/copybara/transform/SkylarkTransformationTest.java diff --git a/javatests/com/google/copybara/transform/TodoReplaceTest.java b/java/javatests/com/google/copybara/transform/TodoReplaceTest.java similarity index 100% rename from javatests/com/google/copybara/transform/TodoReplaceTest.java rename to java/javatests/com/google/copybara/transform/TodoReplaceTest.java diff --git a/javatests/com/google/copybara/transform/VerifyMatchTest.java b/java/javatests/com/google/copybara/transform/VerifyMatchTest.java similarity index 100% rename from javatests/com/google/copybara/transform/VerifyMatchTest.java rename to java/javatests/com/google/copybara/transform/VerifyMatchTest.java diff --git a/javatests/com/google/copybara/transform/debug/BUILD b/java/javatests/com/google/copybara/transform/debug/BUILD similarity index 100% rename from javatests/com/google/copybara/transform/debug/BUILD rename to java/javatests/com/google/copybara/transform/debug/BUILD diff --git a/javatests/com/google/copybara/transform/debug/TransformDebugTest.java b/java/javatests/com/google/copybara/transform/debug/TransformDebugTest.java similarity index 100% rename from javatests/com/google/copybara/transform/debug/TransformDebugTest.java rename to java/javatests/com/google/copybara/transform/debug/TransformDebugTest.java diff --git a/javatests/com/google/copybara/transform/metadata/MetadataModuleTest.java b/java/javatests/com/google/copybara/transform/metadata/MetadataModuleTest.java similarity index 100% rename from javatests/com/google/copybara/transform/metadata/MetadataModuleTest.java rename to java/javatests/com/google/copybara/transform/metadata/MetadataModuleTest.java diff --git a/javatests/com/google/copybara/transform/metadata/RevisionMigratorTest.java b/java/javatests/com/google/copybara/transform/metadata/RevisionMigratorTest.java similarity index 100% rename from javatests/com/google/copybara/transform/metadata/RevisionMigratorTest.java rename to java/javatests/com/google/copybara/transform/metadata/RevisionMigratorTest.java diff --git a/javatests/com/google/copybara/transform/patch/BUILD b/java/javatests/com/google/copybara/transform/patch/BUILD similarity index 100% rename from javatests/com/google/copybara/transform/patch/BUILD rename to java/javatests/com/google/copybara/transform/patch/BUILD diff --git a/javatests/com/google/copybara/transform/patch/PatchTransformationTest.java b/java/javatests/com/google/copybara/transform/patch/PatchTransformationTest.java similarity index 100% rename from javatests/com/google/copybara/transform/patch/PatchTransformationTest.java rename to java/javatests/com/google/copybara/transform/patch/PatchTransformationTest.java diff --git a/javatests/com/google/copybara/transform/patch/PatchingOptionsTest.java b/java/javatests/com/google/copybara/transform/patch/PatchingOptionsTest.java similarity index 100% rename from javatests/com/google/copybara/transform/patch/PatchingOptionsTest.java rename to java/javatests/com/google/copybara/transform/patch/PatchingOptionsTest.java diff --git a/javatests/com/google/copybara/transform/patch/QuiltTransformationTest.java b/java/javatests/com/google/copybara/transform/patch/QuiltTransformationTest.java similarity index 100% rename from javatests/com/google/copybara/transform/patch/QuiltTransformationTest.java rename to java/javatests/com/google/copybara/transform/patch/QuiltTransformationTest.java diff --git a/javatests/com/google/copybara/treestate/TreeStateTest.java b/java/javatests/com/google/copybara/treestate/TreeStateTest.java similarity index 100% rename from javatests/com/google/copybara/treestate/TreeStateTest.java rename to java/javatests/com/google/copybara/treestate/TreeStateTest.java diff --git a/javatests/com/google/copybara/tsjs/npm/BUILD b/java/javatests/com/google/copybara/tsjs/npm/BUILD similarity index 100% rename from javatests/com/google/copybara/tsjs/npm/BUILD rename to java/javatests/com/google/copybara/tsjs/npm/BUILD diff --git a/javatests/com/google/copybara/tsjs/npm/NpmVersionListTest.java b/java/javatests/com/google/copybara/tsjs/npm/NpmVersionListTest.java similarity index 100% rename from javatests/com/google/copybara/tsjs/npm/NpmVersionListTest.java rename to java/javatests/com/google/copybara/tsjs/npm/NpmVersionListTest.java diff --git a/javatests/com/google/copybara/tsjs/npm/NpmVersionResolverTest.java b/java/javatests/com/google/copybara/tsjs/npm/NpmVersionResolverTest.java similarity index 100% rename from javatests/com/google/copybara/tsjs/npm/NpmVersionResolverTest.java rename to java/javatests/com/google/copybara/tsjs/npm/NpmVersionResolverTest.java diff --git a/javatests/com/google/copybara/util/ApplyDestinationPatchTest.java b/java/javatests/com/google/copybara/util/ApplyDestinationPatchTest.java similarity index 100% rename from javatests/com/google/copybara/util/ApplyDestinationPatchTest.java rename to java/javatests/com/google/copybara/util/ApplyDestinationPatchTest.java diff --git a/javatests/com/google/copybara/util/AutoPatchUtilTest.java b/java/javatests/com/google/copybara/util/AutoPatchUtilTest.java similarity index 100% rename from javatests/com/google/copybara/util/AutoPatchUtilTest.java rename to java/javatests/com/google/copybara/util/AutoPatchUtilTest.java diff --git a/javatests/com/google/copybara/util/CommandLineDiffUtilTest.java b/java/javatests/com/google/copybara/util/CommandLineDiffUtilTest.java similarity index 100% rename from javatests/com/google/copybara/util/CommandLineDiffUtilTest.java rename to java/javatests/com/google/copybara/util/CommandLineDiffUtilTest.java diff --git a/javatests/com/google/copybara/util/ConsistencyFileTest.java b/java/javatests/com/google/copybara/util/ConsistencyFileTest.java similarity index 100% rename from javatests/com/google/copybara/util/ConsistencyFileTest.java rename to java/javatests/com/google/copybara/util/ConsistencyFileTest.java diff --git a/javatests/com/google/copybara/util/DiffUtilTest.java b/java/javatests/com/google/copybara/util/DiffUtilTest.java similarity index 100% rename from javatests/com/google/copybara/util/DiffUtilTest.java rename to java/javatests/com/google/copybara/util/DiffUtilTest.java diff --git a/javatests/com/google/copybara/util/DirFactoryTest.java b/java/javatests/com/google/copybara/util/DirFactoryTest.java similarity index 100% rename from javatests/com/google/copybara/util/DirFactoryTest.java rename to java/javatests/com/google/copybara/util/DirFactoryTest.java diff --git a/javatests/com/google/copybara/util/FileUtilTest.java b/java/javatests/com/google/copybara/util/FileUtilTest.java similarity index 100% rename from javatests/com/google/copybara/util/FileUtilTest.java rename to java/javatests/com/google/copybara/util/FileUtilTest.java diff --git a/javatests/com/google/copybara/util/GlobTest.java b/java/javatests/com/google/copybara/util/GlobTest.java similarity index 100% rename from javatests/com/google/copybara/util/GlobTest.java rename to java/javatests/com/google/copybara/util/GlobTest.java diff --git a/javatests/com/google/copybara/util/LimitFilterOutputStreamTest.java b/java/javatests/com/google/copybara/util/LimitFilterOutputStreamTest.java similarity index 100% rename from javatests/com/google/copybara/util/LimitFilterOutputStreamTest.java rename to java/javatests/com/google/copybara/util/LimitFilterOutputStreamTest.java diff --git a/javatests/com/google/copybara/util/MergeImportToolTest.java b/java/javatests/com/google/copybara/util/MergeImportToolTest.java similarity index 100% rename from javatests/com/google/copybara/util/MergeImportToolTest.java rename to java/javatests/com/google/copybara/util/MergeImportToolTest.java diff --git a/javatests/com/google/copybara/util/OriginUtilTest.java b/java/javatests/com/google/copybara/util/OriginUtilTest.java similarity index 100% rename from javatests/com/google/copybara/util/OriginUtilTest.java rename to java/javatests/com/google/copybara/util/OriginUtilTest.java diff --git a/javatests/com/google/copybara/util/RenameDetectorTest.java b/java/javatests/com/google/copybara/util/RenameDetectorTest.java similarity index 100% rename from javatests/com/google/copybara/util/RenameDetectorTest.java rename to java/javatests/com/google/copybara/util/RenameDetectorTest.java diff --git a/javatests/com/google/copybara/util/RepositoryUtilTest.java b/java/javatests/com/google/copybara/util/RepositoryUtilTest.java similarity index 100% rename from javatests/com/google/copybara/util/RepositoryUtilTest.java rename to java/javatests/com/google/copybara/util/RepositoryUtilTest.java diff --git a/javatests/com/google/copybara/util/ScpUtilTest.java b/java/javatests/com/google/copybara/util/ScpUtilTest.java similarity index 100% rename from javatests/com/google/copybara/util/ScpUtilTest.java rename to java/javatests/com/google/copybara/util/ScpUtilTest.java diff --git a/javatests/com/google/copybara/util/SequenceGlobTest.java b/java/javatests/com/google/copybara/util/SequenceGlobTest.java similarity index 100% rename from javatests/com/google/copybara/util/SequenceGlobTest.java rename to java/javatests/com/google/copybara/util/SequenceGlobTest.java diff --git a/javatests/com/google/copybara/util/TablePrinterTest.java b/java/javatests/com/google/copybara/util/TablePrinterTest.java similarity index 100% rename from javatests/com/google/copybara/util/TablePrinterTest.java rename to java/javatests/com/google/copybara/util/TablePrinterTest.java diff --git a/javatests/com/google/copybara/util/console/BUILD b/java/javatests/com/google/copybara/util/console/BUILD similarity index 100% rename from javatests/com/google/copybara/util/console/BUILD rename to java/javatests/com/google/copybara/util/console/BUILD diff --git a/javatests/com/google/copybara/util/console/ConsoleTest.java b/java/javatests/com/google/copybara/util/console/ConsoleTest.java similarity index 100% rename from javatests/com/google/copybara/util/console/ConsoleTest.java rename to java/javatests/com/google/copybara/util/console/ConsoleTest.java diff --git a/javatests/com/google/copybara/util/console/ConsolesTest.java b/java/javatests/com/google/copybara/util/console/ConsolesTest.java similarity index 100% rename from javatests/com/google/copybara/util/console/ConsolesTest.java rename to java/javatests/com/google/copybara/util/console/ConsolesTest.java diff --git a/javatests/com/google/copybara/util/console/DelegateConsoleTest.java b/java/javatests/com/google/copybara/util/console/DelegateConsoleTest.java similarity index 100% rename from javatests/com/google/copybara/util/console/DelegateConsoleTest.java rename to java/javatests/com/google/copybara/util/console/DelegateConsoleTest.java diff --git a/javatests/com/google/copybara/util/console/FileConsoleTest.java b/java/javatests/com/google/copybara/util/console/FileConsoleTest.java similarity index 100% rename from javatests/com/google/copybara/util/console/FileConsoleTest.java rename to java/javatests/com/google/copybara/util/console/FileConsoleTest.java diff --git a/javatests/com/google/copybara/util/console/MultiplexingConsoleTest.java b/java/javatests/com/google/copybara/util/console/MultiplexingConsoleTest.java similarity index 100% rename from javatests/com/google/copybara/util/console/MultiplexingConsoleTest.java rename to java/javatests/com/google/copybara/util/console/MultiplexingConsoleTest.java diff --git a/javatests/com/google/copybara/util/console/NoPromptConsoleTest.java b/java/javatests/com/google/copybara/util/console/NoPromptConsoleTest.java similarity index 100% rename from javatests/com/google/copybara/util/console/NoPromptConsoleTest.java rename to java/javatests/com/google/copybara/util/console/NoPromptConsoleTest.java diff --git a/javatests/com/google/copybara/util/console/testing/BUILD b/java/javatests/com/google/copybara/util/console/testing/BUILD similarity index 100% rename from javatests/com/google/copybara/util/console/testing/BUILD rename to java/javatests/com/google/copybara/util/console/testing/BUILD diff --git a/javatests/com/google/copybara/util/console/testing/TestingConsoleTest.java b/java/javatests/com/google/copybara/util/console/testing/TestingConsoleTest.java similarity index 100% rename from javatests/com/google/copybara/util/console/testing/TestingConsoleTest.java rename to java/javatests/com/google/copybara/util/console/testing/TestingConsoleTest.java diff --git a/javatests/com/google/copybara/version/BUILD b/java/javatests/com/google/copybara/version/BUILD similarity index 100% rename from javatests/com/google/copybara/version/BUILD rename to java/javatests/com/google/copybara/version/BUILD diff --git a/javatests/com/google/copybara/version/CustomVersionSelectorTest.java b/java/javatests/com/google/copybara/version/CustomVersionSelectorTest.java similarity index 100% rename from javatests/com/google/copybara/version/CustomVersionSelectorTest.java rename to java/javatests/com/google/copybara/version/CustomVersionSelectorTest.java diff --git a/javatests/com/google/copybara/version/LatestVersionSelectorTest.java b/java/javatests/com/google/copybara/version/LatestVersionSelectorTest.java similarity index 100% rename from javatests/com/google/copybara/version/LatestVersionSelectorTest.java rename to java/javatests/com/google/copybara/version/LatestVersionSelectorTest.java diff --git a/javatests/com/google/copybara/version/OrderedVersionSelectorTest.java b/java/javatests/com/google/copybara/version/OrderedVersionSelectorTest.java similarity index 100% rename from javatests/com/google/copybara/version/OrderedVersionSelectorTest.java rename to java/javatests/com/google/copybara/version/OrderedVersionSelectorTest.java diff --git a/javatests/com/google/copybara/version/RequestedExactMatchSelectorTest.java b/java/javatests/com/google/copybara/version/RequestedExactMatchSelectorTest.java similarity index 100% rename from javatests/com/google/copybara/version/RequestedExactMatchSelectorTest.java rename to java/javatests/com/google/copybara/version/RequestedExactMatchSelectorTest.java diff --git a/javatests/com/google/copybara/version/RequestedVersionSelectorTest.java b/java/javatests/com/google/copybara/version/RequestedVersionSelectorTest.java similarity index 100% rename from javatests/com/google/copybara/version/RequestedVersionSelectorTest.java rename to java/javatests/com/google/copybara/version/RequestedVersionSelectorTest.java diff --git a/javatests/com/google/copybara/xml/BUILD b/java/javatests/com/google/copybara/xml/BUILD similarity index 100% rename from javatests/com/google/copybara/xml/BUILD rename to java/javatests/com/google/copybara/xml/BUILD diff --git a/javatests/com/google/copybara/xml/XmlTest.java b/java/javatests/com/google/copybara/xml/XmlTest.java similarity index 100% rename from javatests/com/google/copybara/xml/XmlTest.java rename to java/javatests/com/google/copybara/xml/XmlTest.java diff --git a/repositories.bzl b/java/repositories.bzl similarity index 100% rename from repositories.bzl rename to java/repositories.bzl diff --git a/scripts/update_docs b/java/scripts/update_docs similarity index 100% rename from scripts/update_docs rename to java/scripts/update_docs diff --git a/third_party/BUILD b/java/third_party/BUILD similarity index 100% rename from third_party/BUILD rename to java/third_party/BUILD diff --git a/third_party/bazel/BUILD b/java/third_party/bazel/BUILD similarity index 100% rename from third_party/bazel/BUILD rename to java/third_party/bazel/BUILD diff --git a/third_party/bazel/LICENSE b/java/third_party/bazel/LICENSE similarity index 100% rename from third_party/bazel/LICENSE rename to java/third_party/bazel/LICENSE diff --git a/third_party/bazel/bashunit/BUILD b/java/third_party/bazel/bashunit/BUILD similarity index 100% rename from third_party/bazel/bashunit/BUILD rename to java/third_party/bazel/bashunit/BUILD diff --git a/third_party/bazel/bashunit/LICENSE b/java/third_party/bazel/bashunit/LICENSE similarity index 100% rename from third_party/bazel/bashunit/LICENSE rename to java/third_party/bazel/bashunit/LICENSE diff --git a/third_party/bazel/bashunit/testenv.sh b/java/third_party/bazel/bashunit/testenv.sh similarity index 100% rename from third_party/bazel/bashunit/testenv.sh rename to java/third_party/bazel/bashunit/testenv.sh diff --git a/third_party/bazel/bashunit/unittest.bash b/java/third_party/bazel/bashunit/unittest.bash similarity index 100% rename from third_party/bazel/bashunit/unittest.bash rename to java/third_party/bazel/bashunit/unittest.bash diff --git a/third_party/bazel/conditions/BUILD b/java/third_party/bazel/conditions/BUILD similarity index 100% rename from third_party/bazel/conditions/BUILD rename to java/third_party/bazel/conditions/BUILD diff --git a/third_party/bazel/conditions/BUILD.tools b/java/third_party/bazel/conditions/BUILD.tools similarity index 100% rename from third_party/bazel/conditions/BUILD.tools rename to java/third_party/bazel/conditions/BUILD.tools diff --git a/third_party/bazel/main/java/com/google/copybara/shell/AbnormalTerminationException.java b/java/third_party/bazel/main/java/com/google/copybara/shell/AbnormalTerminationException.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/AbnormalTerminationException.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/AbnormalTerminationException.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/BUILD b/java/third_party/bazel/main/java/com/google/copybara/shell/BUILD similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/BUILD rename to java/third_party/bazel/main/java/com/google/copybara/shell/BUILD diff --git a/third_party/bazel/main/java/com/google/copybara/shell/BadExitStatusException.java b/java/third_party/bazel/main/java/com/google/copybara/shell/BadExitStatusException.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/BadExitStatusException.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/BadExitStatusException.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/Command.java b/java/third_party/bazel/main/java/com/google/copybara/shell/Command.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/Command.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/Command.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/CommandException.java b/java/third_party/bazel/main/java/com/google/copybara/shell/CommandException.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/CommandException.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/CommandException.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/CommandResult.java b/java/third_party/bazel/main/java/com/google/copybara/shell/CommandResult.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/CommandResult.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/CommandResult.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/Consumers.java b/java/third_party/bazel/main/java/com/google/copybara/shell/Consumers.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/Consumers.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/Consumers.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/ExecFailedException.java b/java/third_party/bazel/main/java/com/google/copybara/shell/ExecFailedException.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/ExecFailedException.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/ExecFailedException.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/FutureCommandResult.java b/java/third_party/bazel/main/java/com/google/copybara/shell/FutureCommandResult.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/FutureCommandResult.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/FutureCommandResult.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/InputStreamSink.java b/java/third_party/bazel/main/java/com/google/copybara/shell/InputStreamSink.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/InputStreamSink.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/InputStreamSink.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/Killable.java b/java/third_party/bazel/main/java/com/google/copybara/shell/Killable.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/Killable.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/Killable.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/KillableObserver.java b/java/third_party/bazel/main/java/com/google/copybara/shell/KillableObserver.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/KillableObserver.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/KillableObserver.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/LogUtil.java b/java/third_party/bazel/main/java/com/google/copybara/shell/LogUtil.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/LogUtil.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/LogUtil.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/ProcessKillable.java b/java/third_party/bazel/main/java/com/google/copybara/shell/ProcessKillable.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/ProcessKillable.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/ProcessKillable.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/ShellUtils.java b/java/third_party/bazel/main/java/com/google/copybara/shell/ShellUtils.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/ShellUtils.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/ShellUtils.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/SimpleKillableObserver.java b/java/third_party/bazel/main/java/com/google/copybara/shell/SimpleKillableObserver.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/SimpleKillableObserver.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/SimpleKillableObserver.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/TerminationStatus.java b/java/third_party/bazel/main/java/com/google/copybara/shell/TerminationStatus.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/TerminationStatus.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/TerminationStatus.java diff --git a/third_party/bazel/main/java/com/google/copybara/shell/TimeoutKillableObserver.java b/java/third_party/bazel/main/java/com/google/copybara/shell/TimeoutKillableObserver.java similarity index 100% rename from third_party/bazel/main/java/com/google/copybara/shell/TimeoutKillableObserver.java rename to java/third_party/bazel/main/java/com/google/copybara/shell/TimeoutKillableObserver.java diff --git a/third_party/bazel/main/java/com/google/devtools/build/lib/concurrent/BUILD b/java/third_party/bazel/main/java/com/google/devtools/build/lib/concurrent/BUILD similarity index 100% rename from third_party/bazel/main/java/com/google/devtools/build/lib/concurrent/BUILD rename to java/third_party/bazel/main/java/com/google/devtools/build/lib/concurrent/BUILD diff --git a/third_party/bazel/main/java/com/google/devtools/build/lib/concurrent/ThreadSafety.java b/java/third_party/bazel/main/java/com/google/devtools/build/lib/concurrent/ThreadSafety.java similarity index 100% rename from third_party/bazel/main/java/com/google/devtools/build/lib/concurrent/ThreadSafety.java rename to java/third_party/bazel/main/java/com/google/devtools/build/lib/concurrent/ThreadSafety.java diff --git a/third_party/bazel/main/java/net/starlark/java/BUILD b/java/third_party/bazel/main/java/net/starlark/java/BUILD similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/BUILD rename to java/third_party/bazel/main/java/net/starlark/java/BUILD diff --git a/third_party/bazel/main/java/net/starlark/java/annot/BUILD b/java/third_party/bazel/main/java/net/starlark/java/annot/BUILD similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/annot/BUILD rename to java/third_party/bazel/main/java/net/starlark/java/annot/BUILD diff --git a/third_party/bazel/main/java/net/starlark/java/annot/Param.java b/java/third_party/bazel/main/java/net/starlark/java/annot/Param.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/annot/Param.java rename to java/third_party/bazel/main/java/net/starlark/java/annot/Param.java diff --git a/third_party/bazel/main/java/net/starlark/java/annot/ParamType.java b/java/third_party/bazel/main/java/net/starlark/java/annot/ParamType.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/annot/ParamType.java rename to java/third_party/bazel/main/java/net/starlark/java/annot/ParamType.java diff --git a/third_party/bazel/main/java/net/starlark/java/annot/README.md b/java/third_party/bazel/main/java/net/starlark/java/annot/README.md similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/annot/README.md rename to java/third_party/bazel/main/java/net/starlark/java/annot/README.md diff --git a/third_party/bazel/main/java/net/starlark/java/annot/StarlarkAnnotations.java b/java/third_party/bazel/main/java/net/starlark/java/annot/StarlarkAnnotations.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/annot/StarlarkAnnotations.java rename to java/third_party/bazel/main/java/net/starlark/java/annot/StarlarkAnnotations.java diff --git a/third_party/bazel/main/java/net/starlark/java/annot/StarlarkBuiltin.java b/java/third_party/bazel/main/java/net/starlark/java/annot/StarlarkBuiltin.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/annot/StarlarkBuiltin.java rename to java/third_party/bazel/main/java/net/starlark/java/annot/StarlarkBuiltin.java diff --git a/third_party/bazel/main/java/net/starlark/java/annot/StarlarkMethod.java b/java/third_party/bazel/main/java/net/starlark/java/annot/StarlarkMethod.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/annot/StarlarkMethod.java rename to java/third_party/bazel/main/java/net/starlark/java/annot/StarlarkMethod.java diff --git a/third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD b/java/third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD rename to java/third_party/bazel/main/java/net/starlark/java/annot/processor/BUILD diff --git a/third_party/bazel/main/java/net/starlark/java/annot/processor/StarlarkMethodProcessor.java b/java/third_party/bazel/main/java/net/starlark/java/annot/processor/StarlarkMethodProcessor.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/annot/processor/StarlarkMethodProcessor.java rename to java/third_party/bazel/main/java/net/starlark/java/annot/processor/StarlarkMethodProcessor.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/BUILD b/java/third_party/bazel/main/java/net/starlark/java/eval/BUILD similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/BUILD rename to java/third_party/bazel/main/java/net/starlark/java/eval/BUILD diff --git a/third_party/bazel/main/java/net/starlark/java/eval/BuiltinFunction.java b/java/third_party/bazel/main/java/net/starlark/java/eval/BuiltinFunction.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/BuiltinFunction.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/BuiltinFunction.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/CallUtils.java b/java/third_party/bazel/main/java/net/starlark/java/eval/CallUtils.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/CallUtils.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/CallUtils.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/CpuProfiler.java b/java/third_party/bazel/main/java/net/starlark/java/eval/CpuProfiler.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/CpuProfiler.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/CpuProfiler.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/CpuProfilerNativeSupport.java b/java/third_party/bazel/main/java/net/starlark/java/eval/CpuProfilerNativeSupport.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/CpuProfilerNativeSupport.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/CpuProfilerNativeSupport.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/CpuProfilerNativeSupportImpl.java b/java/third_party/bazel/main/java/net/starlark/java/eval/CpuProfilerNativeSupportImpl.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/CpuProfilerNativeSupportImpl.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/CpuProfilerNativeSupportImpl.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Debug.java b/java/third_party/bazel/main/java/net/starlark/java/eval/Debug.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/Debug.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/Debug.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Dict.java b/java/third_party/bazel/main/java/net/starlark/java/eval/Dict.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/Dict.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/Dict.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Eval.java b/java/third_party/bazel/main/java/net/starlark/java/eval/Eval.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/Eval.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/Eval.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/EvalException.java b/java/third_party/bazel/main/java/net/starlark/java/eval/EvalException.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/EvalException.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/EvalException.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/EvalUtils.java b/java/third_party/bazel/main/java/net/starlark/java/eval/EvalUtils.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/EvalUtils.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/EvalUtils.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/FlagGuardedValue.java b/java/third_party/bazel/main/java/net/starlark/java/eval/FlagGuardedValue.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/FlagGuardedValue.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/FlagGuardedValue.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/FormatParser.java b/java/third_party/bazel/main/java/net/starlark/java/eval/FormatParser.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/FormatParser.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/FormatParser.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/GuardedValue.java b/java/third_party/bazel/main/java/net/starlark/java/eval/GuardedValue.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/GuardedValue.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/GuardedValue.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/HasBinary.java b/java/third_party/bazel/main/java/net/starlark/java/eval/HasBinary.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/HasBinary.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/HasBinary.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/ImmutableSingletonStarlarkList.java b/java/third_party/bazel/main/java/net/starlark/java/eval/ImmutableSingletonStarlarkList.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/ImmutableSingletonStarlarkList.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/ImmutableSingletonStarlarkList.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/ImmutableStarlarkList.java b/java/third_party/bazel/main/java/net/starlark/java/eval/ImmutableStarlarkList.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/ImmutableStarlarkList.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/ImmutableStarlarkList.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/JNI.java b/java/third_party/bazel/main/java/net/starlark/java/eval/JNI.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/JNI.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/JNI.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/LazyImmutableStarlarkList.java b/java/third_party/bazel/main/java/net/starlark/java/eval/LazyImmutableStarlarkList.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/LazyImmutableStarlarkList.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/LazyImmutableStarlarkList.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/MethodDescriptor.java b/java/third_party/bazel/main/java/net/starlark/java/eval/MethodDescriptor.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/MethodDescriptor.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/MethodDescriptor.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/MethodLibrary.java b/java/third_party/bazel/main/java/net/starlark/java/eval/MethodLibrary.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/MethodLibrary.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/MethodLibrary.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Module.java b/java/third_party/bazel/main/java/net/starlark/java/eval/Module.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/Module.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/Module.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Mutability.java b/java/third_party/bazel/main/java/net/starlark/java/eval/Mutability.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/Mutability.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/Mutability.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/MutableStarlarkList.java b/java/third_party/bazel/main/java/net/starlark/java/eval/MutableStarlarkList.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/MutableStarlarkList.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/MutableStarlarkList.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/NoneType.java b/java/third_party/bazel/main/java/net/starlark/java/eval/NoneType.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/NoneType.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/NoneType.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/ParamDescriptor.java b/java/third_party/bazel/main/java/net/starlark/java/eval/ParamDescriptor.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/ParamDescriptor.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/ParamDescriptor.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Printer.java b/java/third_party/bazel/main/java/net/starlark/java/eval/Printer.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/Printer.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/Printer.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/RangeList.java b/java/third_party/bazel/main/java/net/starlark/java/eval/RangeList.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/RangeList.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/RangeList.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/RegularImmutableStarlarkList.java b/java/third_party/bazel/main/java/net/starlark/java/eval/RegularImmutableStarlarkList.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/RegularImmutableStarlarkList.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/RegularImmutableStarlarkList.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/RegularTuple.java b/java/third_party/bazel/main/java/net/starlark/java/eval/RegularTuple.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/RegularTuple.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/RegularTuple.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Sequence.java b/java/third_party/bazel/main/java/net/starlark/java/eval/Sequence.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/Sequence.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/Sequence.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/SingletonTuple.java b/java/third_party/bazel/main/java/net/starlark/java/eval/SingletonTuple.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/SingletonTuple.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/SingletonTuple.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Starlark.java b/java/third_party/bazel/main/java/net/starlark/java/eval/Starlark.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/Starlark.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/Starlark.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkCallable.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkCallable.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkCallable.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkCallable.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFloat.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFloat.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkFloat.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFloat.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFunction.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFunction.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkFunction.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkFunction.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIndexable.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIndexable.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkIndexable.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIndexable.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkInt.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkInt.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkInt.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkInt.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIterable.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIterable.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkIterable.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkIterable.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkList.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkList.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkList.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkList.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkMembershipTestable.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkMembershipTestable.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkMembershipTestable.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkMembershipTestable.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSemantics.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSemantics.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkSemantics.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSemantics.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSet.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSet.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkSet.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkSet.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkThread.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkThread.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkThread.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkThread.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkTypeValue.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkTypeValue.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkTypeValue.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkTypeValue.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StarlarkValue.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkValue.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StarlarkValue.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StarlarkValue.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/StringModule.java b/java/third_party/bazel/main/java/net/starlark/java/eval/StringModule.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/StringModule.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/StringModule.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Structure.java b/java/third_party/bazel/main/java/net/starlark/java/eval/Structure.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/Structure.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/Structure.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/SymbolGenerator.java b/java/third_party/bazel/main/java/net/starlark/java/eval/SymbolGenerator.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/SymbolGenerator.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/SymbolGenerator.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/Tuple.java b/java/third_party/bazel/main/java/net/starlark/java/eval/Tuple.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/Tuple.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/Tuple.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/TypeChecker.java b/java/third_party/bazel/main/java/net/starlark/java/eval/TypeChecker.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/TypeChecker.java rename to java/third_party/bazel/main/java/net/starlark/java/eval/TypeChecker.java diff --git a/third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_posix.cc b/java/third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_posix.cc similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_posix.cc rename to java/third_party/bazel/main/java/net/starlark/java/eval/cpu_profiler_posix.cc diff --git a/third_party/bazel/main/java/net/starlark/java/lib/BUILD b/java/third_party/bazel/main/java/net/starlark/java/lib/BUILD similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/lib/BUILD rename to java/third_party/bazel/main/java/net/starlark/java/lib/BUILD diff --git a/third_party/bazel/main/java/net/starlark/java/lib/MapWrapperStructure.java b/java/third_party/bazel/main/java/net/starlark/java/lib/MapWrapperStructure.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/lib/MapWrapperStructure.java rename to java/third_party/bazel/main/java/net/starlark/java/lib/MapWrapperStructure.java diff --git a/third_party/bazel/main/java/net/starlark/java/lib/StarlarkEncodable.java b/java/third_party/bazel/main/java/net/starlark/java/lib/StarlarkEncodable.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/lib/StarlarkEncodable.java rename to java/third_party/bazel/main/java/net/starlark/java/lib/StarlarkEncodable.java diff --git a/third_party/bazel/main/java/net/starlark/java/lib/json/BUILD b/java/third_party/bazel/main/java/net/starlark/java/lib/json/BUILD similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/lib/json/BUILD rename to java/third_party/bazel/main/java/net/starlark/java/lib/json/BUILD diff --git a/third_party/bazel/main/java/net/starlark/java/lib/json/Json.java b/java/third_party/bazel/main/java/net/starlark/java/lib/json/Json.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/lib/json/Json.java rename to java/third_party/bazel/main/java/net/starlark/java/lib/json/Json.java diff --git a/third_party/bazel/main/java/net/starlark/java/spelling/BUILD b/java/third_party/bazel/main/java/net/starlark/java/spelling/BUILD similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/spelling/BUILD rename to java/third_party/bazel/main/java/net/starlark/java/spelling/BUILD diff --git a/third_party/bazel/main/java/net/starlark/java/spelling/SpellChecker.java b/java/third_party/bazel/main/java/net/starlark/java/spelling/SpellChecker.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/spelling/SpellChecker.java rename to java/third_party/bazel/main/java/net/starlark/java/spelling/SpellChecker.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Argument.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Argument.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Argument.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Argument.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/AssignmentStatement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/AssignmentStatement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/AssignmentStatement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/AssignmentStatement.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/BUILD b/java/third_party/bazel/main/java/net/starlark/java/syntax/BUILD similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/BUILD rename to java/third_party/bazel/main/java/net/starlark/java/syntax/BUILD diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/BinaryOperatorExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/BinaryOperatorExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/BinaryOperatorExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/BinaryOperatorExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/CallExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/CallExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/CallExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/CallExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/CastExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/CastExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/CastExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/CastExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Comment.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Comment.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Comment.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Comment.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Comprehension.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Comprehension.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Comprehension.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Comprehension.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ConditionalExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/ConditionalExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/ConditionalExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/ConditionalExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/DefStatement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/DefStatement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/DefStatement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/DefStatement.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/DictExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/DictExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/DictExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/DictExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/DocComments.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/DocComments.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/DocComments.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/DocComments.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/DotExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/DotExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/DotExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/DotExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Ellipsis.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Ellipsis.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Ellipsis.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Ellipsis.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Expression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Expression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Expression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Expression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ExpressionStatement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/ExpressionStatement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/ExpressionStatement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/ExpressionStatement.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/FileLocations.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/FileLocations.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/FileLocations.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/FileLocations.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/FileOptions.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/FileOptions.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/FileOptions.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/FileOptions.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/FloatLiteral.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/FloatLiteral.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/FloatLiteral.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/FloatLiteral.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/FlowStatement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/FlowStatement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/FlowStatement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/FlowStatement.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ForStatement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/ForStatement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/ForStatement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/ForStatement.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Identifier.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Identifier.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Identifier.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Identifier.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/IfStatement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/IfStatement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/IfStatement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/IfStatement.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/IndexExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/IndexExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/IndexExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/IndexExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/IntLiteral.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/IntLiteral.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/IntLiteral.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/IntLiteral.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/IsInstanceExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/IsInstanceExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/IsInstanceExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/IsInstanceExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/LambdaExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/LambdaExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/LambdaExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/LambdaExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Lexer.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Lexer.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Lexer.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Lexer.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ListExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/ListExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/ListExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/ListExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/LoadStatement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/LoadStatement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/LoadStatement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/LoadStatement.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Location.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Location.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Location.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Location.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Node.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Node.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Node.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Node.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/NodePrinter.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/NodePrinter.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/NodePrinter.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/NodePrinter.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/NodeVisitor.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/NodeVisitor.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/NodeVisitor.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/NodeVisitor.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Parameter.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Parameter.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Parameter.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Parameter.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Parser.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Parser.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Parser.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Parser.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ParserInput.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/ParserInput.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/ParserInput.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/ParserInput.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Program.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Program.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Program.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Program.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Resolver.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Resolver.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Resolver.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Resolver.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/ReturnStatement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/ReturnStatement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/ReturnStatement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/ReturnStatement.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/SliceExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/SliceExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/SliceExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/SliceExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/StarlarkFile.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/StarlarkFile.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/StarlarkFile.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/StarlarkFile.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/StarlarkType.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/StarlarkType.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/StarlarkType.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/StarlarkType.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Statement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Statement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Statement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Statement.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/StringLiteral.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/StringLiteral.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/StringLiteral.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/StringLiteral.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/SyntaxError.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/SyntaxError.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/SyntaxError.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/SyntaxError.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/SyntaxUtils.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/SyntaxUtils.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/SyntaxUtils.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/SyntaxUtils.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/TokenKind.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/TokenKind.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/TokenKind.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/TokenKind.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/TypeAliasStatement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/TypeAliasStatement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/TypeAliasStatement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/TypeAliasStatement.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/TypeApplication.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/TypeApplication.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/TypeApplication.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/TypeApplication.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/TypeChecker.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/TypeChecker.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/TypeChecker.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/TypeChecker.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/TypeConstructor.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/TypeConstructor.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/TypeConstructor.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/TypeConstructor.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/TypeContext.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/TypeContext.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/TypeContext.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/TypeContext.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/TypeTagger.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/TypeTagger.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/TypeTagger.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/TypeTagger.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/Types.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/Types.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/Types.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/Types.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/UnaryOperatorExpression.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/UnaryOperatorExpression.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/UnaryOperatorExpression.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/UnaryOperatorExpression.java diff --git a/third_party/bazel/main/java/net/starlark/java/syntax/VarStatement.java b/java/third_party/bazel/main/java/net/starlark/java/syntax/VarStatement.java similarity index 100% rename from third_party/bazel/main/java/net/starlark/java/syntax/VarStatement.java rename to java/third_party/bazel/main/java/net/starlark/java/syntax/VarStatement.java diff --git a/src/Copybara.Cli/ArgParser.cs b/src/Copybara.Cli/ArgParser.cs new file mode 100644 index 000000000..ca94ae93d --- /dev/null +++ b/src/Copybara.Cli/ArgParser.cs @@ -0,0 +1,322 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Globalization; +using System.Reflection; +using Copybara; +using Copybara.Exceptions; + +namespace Copybara.Cli; + +/// +/// A lightweight command-line argument parser that replaces JCommander. It discovers +/// -annotated members on a set of option objects and binds +/// --flag=value / --flag value / boolean flags onto them, collecting anything not +/// recognized as a flag into a list of positional (unnamed) arguments. +/// +/// This is intentionally minimal: it supports the flag surface Copybara actually uses +/// (string, bool, int, , enum-as-string, list and map flags) but does not +/// aim to reproduce every JCommander feature. +/// +public sealed class ArgParser +{ + /// A single registered flag: the member it binds to and the object owning it. + private sealed class Flag + { + public required object Owner { get; init; } + public required MemberInfo Member { get; init; } + public required FlagAttribute Attribute { get; init; } + + public Type MemberType => + Member is PropertyInfo p ? p.PropertyType : ((FieldInfo)Member).FieldType; + + /// Whether this is a "switch" flag that takes no value (unless arity forces one). + public bool IsBooleanSwitch + { + get + { + var t = MemberType; + // A bool flag with explicit arity 1 (e.g. --noprompt=true) still expects a value. + return (t == typeof(bool) || t == typeof(bool?)) && Attribute.Arity != 1; + } + } + + public void SetValue(object? value) + { + switch (Member) + { + case PropertyInfo p: + p.SetValue(Owner, value); + break; + case FieldInfo f: + f.SetValue(Owner, value); + break; + } + } + + public object? GetValue() => + Member is PropertyInfo p ? p.GetValue(Owner) : ((FieldInfo)Member).GetValue(Owner); + } + + private readonly Dictionary _flagsByName = new(StringComparer.Ordinal); + private readonly List _allFlags = new(); + + /// Registers every -annotated member found on the object. + public void AddObject(object optionObject) + { + const BindingFlags bindingFlags = + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance; + + foreach (MemberInfo member in optionObject.GetType() + .GetMembers(bindingFlags) + .Where(m => m is PropertyInfo or FieldInfo)) + { + var attr = member.GetCustomAttribute(inherit: true); + if (attr == null) + { + continue; + } + + var flag = new Flag { Owner = optionObject, Member = member, Attribute = attr }; + _allFlags.Add(flag); + foreach (string name in attr.Names) + { + _flagsByName[name] = flag; + } + } + } + + /// Registers every option in the collection. + public void AddObjects(IEnumerable optionObjects) + { + foreach (var o in optionObjects) + { + AddObject(o); + } + } + + /// All registered flag names, used for "did you mean" hints and usage output. + public IReadOnlyList AllFlagNames => + _allFlags.SelectMany(f => f.Attribute.Names).Distinct().OrderBy(n => n, StringComparer.Ordinal) + .ToImmutableArray(); + + /// Descriptions of all non-hidden flags for usage output. + public IReadOnlyList<(string Names, string Description)> Descriptions => + _allFlags.Where(f => !f.Attribute.Hidden) + .Select(f => (string.Join(", ", f.Attribute.Names), f.Attribute.Description)) + .ToImmutableArray(); + + /// + /// Parses , binding recognized flags onto the registered option objects + /// and returning the positional (unnamed) arguments in order. + /// + /// on unknown flags or malformed values. + public IReadOnlyList Parse(string[] args) + { + var unnamed = ImmutableArray.CreateBuilder(); + int i = 0; + while (i < args.Length) + { + string arg = args[i]; + + // Only tokens beginning with '-' are candidate flags. Everything else is positional. + if (arg.Length < 2 || arg[0] != '-') + { + unnamed.Add(arg); + i++; + continue; + } + + string name = arg; + string? inlineValue = null; + int eq = arg.IndexOf('='); + if (eq >= 0) + { + name = arg.Substring(0, eq); + inlineValue = arg.Substring(eq + 1); + } + + if (!_flagsByName.TryGetValue(name, out Flag? flag)) + { + // Not a known flag. Treat as positional so Main can warn about likely typos, matching + // upstream's behavior of surfacing "looks like a flag" arguments to the command. + unnamed.Add(arg); + i++; + continue; + } + + if (flag.IsBooleanSwitch && inlineValue == null) + { + flag.SetValue(true); + i++; + continue; + } + + string rawValue; + if (inlineValue != null) + { + rawValue = inlineValue; + i++; + } + else + { + if (i + 1 >= args.Length) + { + throw new CommandLineException($"Missing value for flag '{name}'."); + } + rawValue = args[i + 1]; + i += 2; + } + + AssignValue(flag, name, rawValue); + } + + return unnamed.ToImmutable(); + } + + private static void AssignValue(Flag flag, string name, string rawValue) + { + Type type = flag.MemberType; + Type target = Nullable.GetUnderlyingType(type) ?? type; + + try + { + if (target == typeof(string)) + { + flag.SetValue(rawValue); + } + else if (target == typeof(bool)) + { + flag.SetValue(ParseBool(rawValue)); + } + else if (target == typeof(int)) + { + flag.SetValue(int.Parse(rawValue, CultureInfo.InvariantCulture)); + } + else if (target == typeof(long)) + { + flag.SetValue(long.Parse(rawValue, CultureInfo.InvariantCulture)); + } + else if (target == typeof(TimeSpan)) + { + flag.SetValue(DurationConverter.Convert(rawValue)); + } + else if (target.IsEnum) + { + flag.SetValue(Enum.Parse(target, rawValue, ignoreCase: true)); + } + else if (IsListType(target, out Type? elementType)) + { + AppendToList(flag, rawValue, elementType!); + } + else if (IsDictType(target)) + { + AssignDict(flag, rawValue); + } + else + { + throw new CommandLineException( + $"Unsupported flag type '{type}' for flag '{name}'."); + } + } + catch (CommandLineException) + { + throw; + } + catch (Exception e) + { + throw new CommandLineException($"Invalid value '{rawValue}' for flag '{name}': {e.Message}"); + } + } + + private static bool ParseBool(string value) + { + if (bool.TryParse(value, out bool b)) + { + return b; + } + throw new CommandLineException($"Expected 'true' or 'false' but got '{value}'."); + } + + private static bool IsListType(Type type, out Type? elementType) + { + if (type.IsGenericType) + { + Type def = type.GetGenericTypeDefinition(); + if (def == typeof(List<>) || def == typeof(IReadOnlyList<>) + || def == typeof(IList<>) || def == typeof(ImmutableArray<>)) + { + elementType = type.GetGenericArguments()[0]; + return true; + } + } + elementType = null; + return false; + } + + private static bool IsDictType(Type type) => + type.IsGenericType + && (type.GetGenericTypeDefinition() == typeof(ImmutableDictionary<,>) + || type.GetGenericTypeDefinition() == typeof(Dictionary<,>)); + + private static void AppendToList(Flag flag, string rawValue, Type elementType) + { + // List flags accept comma-separated values, matching JCommander's default splitter. + var pieces = rawValue.Split(',', StringSplitOptions.RemoveEmptyEntries); + var current = flag.GetValue(); + var listType = typeof(List<>).MakeGenericType(elementType); + var list = (System.Collections.IList)Activator.CreateInstance(listType)!; + if (current is System.Collections.IEnumerable existing and not string) + { + foreach (var item in existing) + { + list.Add(item); + } + } + foreach (var piece in pieces) + { + list.Add(elementType == typeof(string) + ? piece + : Convert.ChangeType(piece, elementType, CultureInfo.InvariantCulture)); + } + flag.SetValue(list); + } + + private static void AssignDict(Flag flag, string rawValue) + { + // Map flags are 'k1:v1,k2:v2'. Only is used in practice. + var builder = ImmutableDictionary.CreateBuilder(); + if (flag.GetValue() is ImmutableDictionary existing) + { + foreach (var kv in existing) + { + builder[kv.Key] = kv.Value; + } + } + foreach (var entry in rawValue.Split(',', StringSplitOptions.RemoveEmptyEntries)) + { + int colon = entry.IndexOf(':'); + if (colon < 0) + { + throw new CommandLineException( + $"Expected 'key:value' pairs but got '{entry}'."); + } + builder[entry.Substring(0, colon)] = entry.Substring(colon + 1); + } + flag.SetValue(builder.ToImmutable()); + } +} diff --git a/src/Copybara.Cli/CommandEnv.cs b/src/Copybara.Cli/CommandEnv.cs new file mode 100644 index 000000000..67a3a1ea2 --- /dev/null +++ b/src/Copybara.Cli/CommandEnv.cs @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara; +using Copybara.Common; +using Copybara.Exceptions; + +namespace Copybara.Cli; + +/// +/// Environment information for command execution: arguments, workdir, etc. +/// +public sealed class CommandEnv +{ + private readonly string _workdir; + private readonly Options _options; + private readonly MainArguments? _mainArgs; + private readonly ImmutableArray _args; + private ConfigFileArgs? _configFileArgs; + + public CommandEnv( + string workdir, Options options, IEnumerable args, MainArguments? mainArgs) + { + _workdir = Preconditions.CheckNotNull(workdir); + _options = Preconditions.CheckNotNull(options); + _args = args.ToImmutableArray(); + _mainArgs = mainArgs; + } + + /// + /// Instantiate a new CommandEnv. Meant for use with construction of new ICopybaraCmd objects. + /// + public CommandEnv(string workdir, Options options, IEnumerable args) + : this(workdir, options, args, null) + { + } + + /// + /// Get the arguments parsed as config [migration [source_ref]...] if the command uses that + /// format. + /// + public ConfigFileArgs? GetConfigFileArgs() => _configFileArgs; + + public MainArguments? GetMainArgs() => _mainArgs; + + /// Parse the CLI arguments as config [workflow [source_ref]...]. + /// + public ConfigFileArgs ParseConfigFileArgs(ICopybaraCmd cmd, bool usesSourceRef) + { + Preconditions.CheckState( + _configFileArgs == null, + "ParseConfigFileArgs was already called. Only one invocation allowed."); + if (_args.IsDefaultOrEmpty) + { + throw new CommandLineException( + $"Configuration file missing for '{cmd.Name}' subcommand."); + } + + string configPath = _args[0]; + + if (_args.Length < 2) + { + _configFileArgs = new ConfigFileArgs(configPath, workflowName: null); + return _configFileArgs; + } + + string workflowName = _args[1]; + if (_args.Length < 3) + { + _configFileArgs = new ConfigFileArgs(configPath, workflowName); + return _configFileArgs; + } + + if (!usesSourceRef) + { + throw new CommandLineException( + $"Too many arguments for subcommand '{cmd.Name}'"); + } + + _configFileArgs = new ConfigFileArgs( + configPath, workflowName, _args.Skip(2)); + return _configFileArgs; + } + + public string GetWorkdir() => _workdir; + + public Options GetOptions() => _options; + + public IReadOnlyList GetArgs() => _args; +} diff --git a/src/Copybara.Cli/Copybara.Cli.csproj b/src/Copybara.Cli/Copybara.Cli.csproj new file mode 100644 index 000000000..018245c3b --- /dev/null +++ b/src/Copybara.Cli/Copybara.Cli.csproj @@ -0,0 +1,26 @@ + + + Exe + Copybara.Cli + copybara + true + copybara + Copybara + ./nupkg + 0.1.0 + A tool for transforming and moving code between repositories (.NET port of Google Copybara). + icon.png + README.md + copybara;git;migration;vcs;starlark;code-transformation + + + + + + + + + + + + diff --git a/src/Copybara.Cli/DurationConverter.cs b/src/Copybara.Cli/DurationConverter.cs new file mode 100644 index 000000000..83b05ba20 --- /dev/null +++ b/src/Copybara.Cli/DurationConverter.cs @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Globalization; +using Copybara.Exceptions; + +namespace Copybara.Cli; + +/// +/// Converts strings like 10s/10m/10h/10d to a . +/// Port of com.google.copybara.jcommander.DurationConverter. +/// +public static class DurationConverter +{ + public static TimeSpan Convert(string value) + { + if (value.Length < 2) + { + throw DurationException(value); + } + + if (!int.TryParse( + value.AsSpan(0, value.Length - 1), + NumberStyles.Integer, + CultureInfo.InvariantCulture, + out int num) + || num < 0) + { + throw DurationException(value); + } + + char unit = value[^1]; + return unit switch + { + 's' => TimeSpan.FromSeconds(num), + 'm' => TimeSpan.FromMinutes(num), + 'h' => TimeSpan.FromHours(num), + 'd' => TimeSpan.FromDays(num), + _ => throw DurationException(value), + }; + } + + private static CommandLineException DurationException(string value) => + new(string.Format( + "Invalid value for duration '{0}', valid value examples: 10s, 10m, 10h or 10d", value)); +} diff --git a/src/Copybara.Cli/FuncConfigLoaderProvider.cs b/src/Copybara.Cli/FuncConfigLoaderProvider.cs new file mode 100644 index 000000000..118aad76d --- /dev/null +++ b/src/Copybara.Cli/FuncConfigLoaderProvider.cs @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara; +using Copybara.Config; + +namespace Copybara.Cli; + +/// +/// A backed by a delegate. Mirrors the lambda-based +/// implementation returned by upstream's Main.newConfigLoaderProvider. +/// +/// Note: this assumes the config agent defines ConfigLoaderProvider in the +/// Copybara namespace as an interface exposing +/// ConfigLoader NewLoader(string configPath, string? sourceRef) (faithful to Java's +/// ConfigLoaderProvider.newLoader). If it lands as a delegate instead, this adapter should +/// be replaced during consolidation. +/// +public sealed class FuncConfigLoaderProvider : IConfigLoaderProvider +{ + private readonly Func _factory; + + public FuncConfigLoaderProvider(Func factory) + { + _factory = factory; + } + + public ConfigLoader NewLoader(string configPath, string? sourceRef) => + _factory(configPath, sourceRef); +} diff --git a/src/Copybara.Cli/ICopybaraCmd.cs b/src/Copybara.Cli/ICopybaraCmd.cs new file mode 100644 index 000000000..9b8bb0790 --- /dev/null +++ b/src/Copybara.Cli/ICopybaraCmd.cs @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Util; + +namespace Copybara.Cli; + +/// +/// A Copybara command like 'info', 'migrate', etc. +/// +public interface ICopybaraCmd +{ + /// Run the command. + /// Command environment: params, workdir, etc. + /// Result exit code. + /// + /// + /// + ExitCode Run(CommandEnv commandEnv); + + /// Command name. + string Name { get; } +} diff --git a/src/Copybara.Cli/InfoCmd.cs b/src/Copybara.Cli/InfoCmd.cs new file mode 100644 index 000000000..53e654c98 --- /dev/null +++ b/src/Copybara.Cli/InfoCmd.cs @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara; +using Copybara.Common; +using Copybara.Config; +using Copybara.Revision; +using Copybara.Util; +using Starlark.Eval; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Cli; + +/// +/// Reads the last migrated revision in the origin and destination. +/// +public sealed class InfoCmd : ICopybaraCmd +{ + private const int RevisionMaxLength = 15; + private const int DescriptionMaxLength = 80; + private const int AuthorMaxLength = 40; + private const string DateFormat = "yyyy-MM-dd HH:mm:ss"; + + private readonly IConfigLoaderProvider _configLoaderProvider; + private readonly IContextProvider _contextProvider; + + public InfoCmd(IConfigLoaderProvider configLoaderProvider, IContextProvider contextProvider) + { + _configLoaderProvider = Preconditions.CheckNotNull(configLoaderProvider); + _contextProvider = Preconditions.CheckNotNull(contextProvider); + } + + public ExitCode Run(CommandEnv commandEnv) + { + ConfigFileArgs configFileArgs = commandEnv.GetConfigFileArgs()!; + GeneralOptions generalOptions = commandEnv.GetOptions().Get(); + Console console = generalOptions.GetConsole(); + bool includeDefinitions = generalOptions.InfoIncludeDefinition; + ConfigWithDependencies config = _configLoaderProvider + .NewLoader(configFileArgs.GetConfigPath(), configFileArgs.GetSourceRef()) + .LoadWithDependencies(console); + + if (generalOptions.InfoListOnly) + { + ListMigrations(commandEnv, config.GetConfig(), includeDefinitions); + return ExitCode.Success; + } + + if (configFileArgs.HasWorkflowName()) + { + IReadOnlyDictionary context = _contextProvider.GetContext( + config, configFileArgs, _configLoaderProvider, commandEnv.GetOptions(), console); + bool hasAvailableChanges = + InfoWithFailureHandling( + commandEnv.GetOptions(), + config.GetConfig(), + configFileArgs.GetWorkflowName(), + context); + return hasAvailableChanges ? ExitCode.Success : ExitCode.NoOp; + } + + ShowAllMigrations(commandEnv, config.GetConfig(), includeDefinitions); + return ExitCode.Success; + } + + private static string GetShortFileName(string? path) + { + if (string.IsNullOrEmpty(path)) + { + return ""; + } + int idx = path.LastIndexOf('/'); + return idx >= 0 ? path.Substring(idx + 1) : path; + } + + private static string FormatStackEntry( + ImmutableArray callStack, int stackIndex, bool extraSpacing) + { + Preconditions.CheckArgument(stackIndex > 0, "Index must be greater than 0"); + string definitionName = callStack[stackIndex].Name; + int line = callStack[stackIndex - 1].Location.Line; + if (line != 0) + { + string callerFile = GetShortFileName(callStack[stackIndex - 1].Location.File); + string mainFile = GetShortFileName(callStack[0].Location.File); + if (!callerFile.Equals(mainFile, StringComparison.Ordinal)) + { + string spacing = extraSpacing ? " " : ""; + return $"{definitionName}@{line}{spacing}[{callerFile}]"; + } + return $"{definitionName}@{line}"; + } + return definitionName; + } + + private static void ListMigrations( + CommandEnv commandEnv, Config.Config config, bool includeDefinitions) + { + Console console = commandEnv.GetOptions().Get().GetConsole(); + if (includeDefinitions) + { + var entries = new List(); + foreach (string name in config.GetMigrations().Keys.OrderBy(n => n, StringComparer.Ordinal)) + { + IMigration m = config.GetMigration(name); + var callStack = m.GetDefinitionStack(); + if (!callStack.IsDefaultOrEmpty && callStack.Length > 1) + { + var fullStack = new System.Text.StringBuilder(FormatStackEntry(callStack, 1, false)); + for (int i = 2; i < callStack.Length; i++) + { + fullStack.Append("->").Append(FormatStackEntry(callStack, i, false)); + } + entries.Add(name + ":" + fullStack); + } + } + console.InfoFmt("MIGRATIONS+DEFINITIONSTACK: %s", string.Join(",", entries)); + } + else + { + console.InfoFmt( + "MIGRATIONS: %s", + string.Join( + ",", config.GetMigrations().Keys.OrderBy(n => n, StringComparer.Ordinal))); + } + } + + private static void ShowAllMigrations( + CommandEnv commandEnv, Config.Config config, bool includeDefinitions) + { + TablePrinter table; + var sortedMigrations = config.GetMigrations().Values + .OrderBy(m => m.GetName(), StringComparer.Ordinal) + .ToImmutableArray(); + + if (includeDefinitions) + { + table = new TablePrinter("Name", "Definition", "Description"); + foreach (IMigration m in sortedMigrations) + { + var callStack = m.GetDefinitionStack(); + if (!callStack.IsDefaultOrEmpty && callStack.Length > 1) + { + table.AddRow( + m.GetName(), + FormatStackEntry(callStack, 1, true), + m.GetDescription() ?? ""); + for (int i = 2; i < callStack.Length; i++) + { + table.AddRow("", "↳ " + FormatStackEntry(callStack, i, true), ""); + } + } + } + } + else + { + table = new TablePrinter("Name", "Origin", "Destination", "Mode", "Description"); + foreach (IMigration m in sortedMigrations) + { + table.AddRow( + m.GetName(), + PrettyOriginDestination(m.GetOriginDescription()), + PrettyOriginDestination(m.GetDestinationDescription()), + m.GetModeString(), + m.GetDescription() ?? ""); + } + } + + Console console = commandEnv.GetOptions().Get().GetConsole(); + foreach (string line in table.Build()) + { + console.Info(line); + } + console.Info( + "To get information about the state of any migration run:\n\n" + + " copybara info " + config.GetLocation() + " [workflow_name]" + + "\n"); + } + + private static string PrettyOriginDestination(ImmutableListMultimap desc) + { + string type = desc["type"].Single(); + var urls = desc["url"]; + return type + (urls.Length > 0 ? " (" + urls[0] + ")" : ""); + } + + /// Retrieves the info of the migration and prints it to the console. + private static bool InfoWithFailureHandling( + Options options, Config.Config config, string migrationName, + IReadOnlyDictionary context) + { + // TODO(port): dispatch InfoFailedEvent via eventMonitors() once the monitor package is + // ported. Currently only the info body is executed. + _ = context; + return Info(options, config, migrationName); + } + + private static bool Info(Options options, Config.Config config, string migrationName) + { + Info info = GetInfo(migrationName, config); + Console console = options.Get().GetConsole(); + int outputSize = 0; + bool hasAvailableChanges = false; + foreach (MigrationReference migrationRef in info.MigrationReferences) + { + console.Info(string.Format( + "'{0}': last_migrated {1} - last_available {2}.", + migrationRef.GetLabel(), + migrationRef.LastMigrated != null ? migrationRef.LastMigrated.AsString() : "None", + migrationRef.GetLastAvailableToMigrate() != null + ? migrationRef.GetLastAvailableToMigrate()!.AsString() + : "None")); + + var availableToMigrate = migrationRef.GetAvailableToMigrate(); + int outputLimit = options.Get().GetOutputLimit(); + if (availableToMigrate.Count > 0) + { + hasAvailableChanges = true; + console.InfoFmt( + "Available changes %s:", + availableToMigrate.Count <= outputLimit + ? $"({availableToMigrate.Count})" + : $"(showing only first {outputLimit} out of {availableToMigrate.Count})"); + var table = new TablePrinter("Date", "Revision", "Description", "Author"); + foreach (var change in availableToMigrate.Take(outputLimit)) + { + outputSize++; + table.AddRow( + change.GetDateTime().ToString(DateFormat), + Truncate(change.GetRevision().AsString(), RevisionMaxLength, ""), + Truncate(change.FirstLineMessage(), DescriptionMaxLength, "..."), + Truncate(change.GetAuthor().ToString(), AuthorMaxLength, "...")); + } + foreach (string line in table.Build()) + { + console.Info(line); + } + } + if (outputSize > 100) + { + console.InfoFmt( + "Use %s to limit the output of the command.", GeneralOptions.OutputLimitFlag); + } + } + + // TODO(port): dispatch InfoFinishedEvent via eventMonitors() once the monitor package is + // ported. + return hasAvailableChanges; + } + + private static Info GetInfo(string migrationName, Config.Config config) => + config.GetMigration(migrationName).GetInfo(); + + /// Truncates to , appending the truncation + /// indicator (equivalent to Guava's Ascii.truncate). + private static string Truncate(string value, int maxLength, string indicator) + { + if (value.Length <= maxLength) + { + return value; + } + int truncationLength = maxLength - indicator.Length; + return value.Substring(0, Math.Max(0, truncationLength)) + indicator; + } + + public string Name => "info"; +} diff --git a/src/Copybara.Cli/Main.cs b/src/Copybara.Cli/Main.cs new file mode 100644 index 000000000..f04f2bca4 --- /dev/null +++ b/src/Copybara.Cli/Main.cs @@ -0,0 +1,565 @@ +/* + * Copyright (C) 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara; +using Copybara.Common; +using Copybara.Config; +using Copybara.Exceptions; +using Copybara.Profiler; +using Copybara.Util; +using Copybara.Util.Console; +using Microsoft.Extensions.Logging; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Cli; + +/// +/// Main class that invokes Copybara from the command line. +/// +/// This class should only know about how to validate and parse command-line arguments in order +/// to invoke Copybara. +/// +public class Main +{ + public const string BuildLabel = "Build label"; + + // These flags are read before the arg parser is initialized, because of the console lifecycle. + // They mirror the (internal) constants on GeneralOptions. + private const string ConsoleFilePathFlag = "--console-file-path"; + private const string ConsoleFileFlushIntervalFlag = "--console-file-flush-interval"; + private static readonly TimeSpan DefaultConsoleFileFlushInterval = TimeSpan.FromSeconds(30); + + /// + /// Commands whose config-file arguments should be parsed, mapping to whether they consume a + /// source ref. + /// + private static readonly ImmutableDictionary + CommandNamesThatUseConfigFilesToUseSourceRef = + new Dictionary + { + ["migrate"] = true, + ["info"] = false, + ["validate"] = false, + }.ToImmutableDictionary(); + + /// The environment, typically the process environment variables. Injected for tests. + protected readonly IReadOnlyDictionary Environment; + + protected Profiler.Profiler? Profiler; + protected ArgParser? ArgParser; + + private Console? _console; + + public Main() + : this(GetSystemEnvironment()) + { + } + + public Main(IReadOnlyDictionary environment) + { + Environment = Preconditions.CheckNotNull(environment); + } + + public static int Main_(string[] args) => (int)new Main().Run(args); + + public ExitCode Run(string[] args) + { + // We need a console before parsing the args because it could fail with wrong arguments and + // we need to show the error. + _console = GetConsole(args); + Console console = _console; + + console.StartupMessage(GetVersion()); + console.VerboseFmt("Running: %s", string.Join(' ', args)); + + CommandResult result = RunInternal(args, console); + try + { + Shutdown(result); + } + catch (Exception e) + { + HandleUnexpectedError(console, "Execution was interrupted.", args, e); + } + + return result.ExitCode; + } + + /// Finds out about verbose output before the arg parser has been initialized. + protected static bool IsVerbose(string[] args) => + args.Any(s => s == "-v" || s == "--verbose"); + + /// Finds out if logging is enabled before the arg parser has been initialized. + protected static bool IsEnableLogging(string[] args) => !args.Contains("--nologging"); + + /// + /// Finds a flag value before the arg parser is initialized. Returns null if the flag is not + /// present or has no value ('=' and ' ' accepted as separators). Does not support arity 0 flags. + /// + protected static string? FindFlagValue(string[] args, string flagName) + { + for (int index = 0; index < args.Length; index++) + { + if (args[index] == flagName) + { + if (index < args.Length - 1 && !args[index + 1].StartsWith('-')) + { + return args[index + 1]; + } + return null; + } + if (args[index].StartsWith(flagName + "=", StringComparison.Ordinal)) + { + return args[index].Substring(flagName.Length + 1); + } + } + return null; + } + + /// The exit code and the command executed. + protected sealed record CommandResult( + ExitCode ExitCode, ICopybaraCmd? Command, CommandEnv? CommandEnv); + + /// + /// Runs the command and returns the . Also responsible for the exception + /// handling/logging. + /// + private CommandResult RunInternal(string[] args, Console console) + { + CommandEnv? commandEnv = null; + ICopybaraCmd? subcommand = null; + + try + { + ModuleSet moduleSet = NewModuleSet(Environment, console); + + var mainArgs = new MainArguments(args); + Options options = moduleSet.GetOptions(); + + ArgParser = new ArgParser(); + ArgParser.AddObjects(options.GetAll().Cast()); + ArgParser.AddObject(mainArgs); + mainArgs.Unnamed = ArgParser.Parse(args).ToList(); + + string version = GetVersion(); + + IConfigLoaderProvider configLoaderProvider = NewConfigLoaderProvider(moduleSet); + + ImmutableDictionary commands = + GetCommands(moduleSet, configLoaderProvider) + .ToImmutableDictionary(c => c.Name, c => c); + + MainArguments.CommandWithArgs cmdToRun = + mainArgs.ParseCommand(commands, commands["migrate"]); + subcommand = cmdToRun.Subcommand; + + WarnAboutPossibleFlags(cmdToRun, console); + + InitEnvironment(options, cmdToRun.Subcommand, args); + + GeneralOptions generalOptions = options.Get(); + string baseWorkdir = mainArgs.GetBaseWorkdir(generalOptions, generalOptions.GetFileSystem()); + + commandEnv = new CommandEnv(baseWorkdir, options, cmdToRun.Args, mainArgs); + if (CommandNamesThatUseConfigFilesToUseSourceRef.TryGetValue(subcommand.Name, out bool useSourceRef)) + { + commandEnv.ParseConfigFileArgs(subcommand, useSourceRef); + } + + console.VerboseFmt("Current working directory: %s", generalOptions.GetCwd()); + generalOptions.GetConsole().ProgressFmt("Running %s", subcommand.Name); + + ExitCode exitCode = subcommand.Run(commandEnv); + return new CommandResult(exitCode, subcommand, commandEnv); + } + catch (CommandLineException e) + { + Consoles.PrintCauseChain(LogLevel.Warning, console, args, e); + console.Error("Try 'copybara help'."); + return new CommandResult(ExitCode.CommandLineError, subcommand, commandEnv); + } + catch (EmptyChangeException e) + { + // This is not necessarily an error. Maybe the tool was run previously and there are no + // new changes to import. (EmptyChangeException derives from ValidationException, so this + // must be caught before ValidationException.) + console.Warn(e.Message); + return new CommandResult(ExitCode.NoOp, subcommand, commandEnv); + } + catch (ValidationException e) + { + Consoles.PrintCauseChain(LogLevel.Warning, console, args, e); + return new CommandResult(ExitCode.ConfigurationError, subcommand, commandEnv); + } + catch (RepoException e) + { + Consoles.PrintCauseChain(LogLevel.Error, console, args, e); + if (e.InnerException is OperationCanceledException) + { + return new CommandResult(ExitCode.Interrupted, subcommand, commandEnv); + } + return new CommandResult(ExitCode.RepositoryError, subcommand, commandEnv); + } + catch (IOException e) + { + HandleUnexpectedError(console, e.Message, args, e); + return new CommandResult(ExitCode.EnvironmentError, subcommand, commandEnv); + } + catch (Exception e) + { + // This usually indicates a serious programming error that will require Copybara team + // intervention. Print stack trace without concern for presentation. + System.Console.Error.WriteLine(e); + HandleUnexpectedError(console, "Unexpected error: " + e.Message, args, e); + return new CommandResult(ExitCode.InternalError, subcommand, commandEnv); + } + } + + private void WarnAboutPossibleFlags(MainArguments.CommandWithArgs cmdToRun, Console console) + { + var possibleFlags = cmdToRun.Args + .Where(arg => arg.StartsWith("--", StringComparison.Ordinal)) + .ToImmutableArray(); + if (possibleFlags.IsEmpty) + { + return; + } + + IReadOnlyList allNames = ArgParser?.AllFlagNames ?? Array.Empty(); + foreach (string possibleFlag in possibleFlags) + { + var candidates = allNames + .Where(s => FlagDistance(s, possibleFlag) <= 1) + .OrderBy(s => s, StringComparer.Ordinal) + .Distinct() + .ToImmutableArray(); + if (candidates.IsEmpty) + { + console.WarnFmt( + "Argument '%s' looks like a flag, but was not parsed as one, is this" + + " intentional?", + possibleFlag); + } + else + { + console.WarnFmt( + "Argument '%s' looks like a flag, but was not parsed as one, did you mean one" + + " of %s?", + possibleFlag, "[" + string.Join(", ", candidates) + "]"); + } + } + } + + /// Naive algorithm to propose similar flags (dropped pre-/suffixes). + private static int FlagDistance(string flag, string input) + { + var flagSet = flag.Split('_', '-').Select(s => s.ToLowerInvariant()).ToHashSet(); + var inputSet = input.Split('_', '-').Select(s => s.ToLowerInvariant()).ToHashSet(); + return inputSet.Count - inputSet.Intersect(flagSet).Count(); + } + + public IReadOnlyList GetCommands( + ModuleSet moduleSet, IConfigLoaderProvider configLoaderProvider) + { + ConfigValidator validator = GetConfigValidator(moduleSet.GetOptions()); + Action consumer = GetMigrationRanConsumer(); + // TODO(port): OnboardCmd, GeneratorCmd, RegenerateCmd are not ported yet and are omitted. + return new ICopybaraCmd[] + { + new MigrateCmd(validator, consumer, configLoaderProvider, moduleSet), + new InfoCmd(configLoaderProvider, NewInfoContextProvider()), + new ValidateCmd(validator, consumer, configLoaderProvider), + new HelpCmd(this), + new VersionCmd(this), + }; + } + + /// Returns a short string representing the version of the binary. + protected virtual string GetVersion() + { + var buildInfo = GetBuildInfo(); + return buildInfo.TryGetValue(BuildLabel, out var label) ? label : "Unknown version"; + } + + private static ImmutableDictionary GetBuildInfo() + { + // TODO(port): upstream loads /build-data.properties from resources. Not wired up yet. + return ImmutableDictionary.Empty; + } + + /// Returns a string describing who and when the binary was built. + protected virtual string GetBinaryInfo() => + string.Join("\n", GetBuildInfo().Select(kv => $"{kv.Key}: {kv.Value}")); + + protected virtual Action GetMigrationRanConsumer() => _ => { }; + + protected virtual ConfigValidator GetConfigValidator(Options options) => + new DefaultConfigValidator(); + + private sealed class DefaultConfigValidator : ConfigValidator + { + } + + /// Returns a new module set. + protected virtual ModuleSet NewModuleSet( + IReadOnlyDictionary environment, Console console) + { + string fsRoot = Path.GetPathRoot(Directory.GetCurrentDirectory()) ?? "/"; + return new ModuleSupplier(environment, fsRoot, console).Create(); + } + + protected virtual IConfigLoaderProvider NewConfigLoaderProvider(ModuleSet moduleSet) + { + GeneralOptions generalOptions = moduleSet.GetOptions().Get(); + return new FuncConfigLoaderProvider((configPath, sourceRef) => + new ConfigLoader( + moduleSet, + CreateConfigFileWithHeuristic( + ValidateLocalConfig(generalOptions, configPath), + generalOptions.GetConfigRoot()), + generalOptions.GetStarlarkMode())); + } + + protected virtual IContextProvider NewInfoContextProvider() => new InfoContextProvider(); + + private sealed class InfoContextProvider : IContextProvider + { + public IReadOnlyDictionary GetContext( + Copybara.Config.Config config, + ConfigFileArgs configFileArgs, + IConfigLoaderProvider configLoaderProvider, + Console console) => + new Dictionary { ["copybara_config"] = config.GetLocation() }; + } + + /// + /// Validates that the passed config file is correct (exists, right filename, etc.) and returns + /// its absolute path. + /// + /// + /// + protected virtual string ValidateLocalConfig(GeneralOptions generalOptions, string configLocation) + { + string configPath = Path.GetFullPath(configLocation); + string? fileName = Path.GetFileName(configPath); + ValidationException.CheckCondition( + !string.IsNullOrEmpty(fileName), + "The configuration path '{0}' is not a file.", + configPath); + ValidationException.CheckCondition( + fileName == MainArguments.CopybaraSkylarkConfigFilename, + "Copybara config file filename should be '{0}' but it is '{1}'.", + MainArguments.CopybaraSkylarkConfigFilename, + fileName!); + + if (!File.Exists(configPath)) + { + throw new CommandLineException("Configuration file not found: " + configPath); + } + return configPath; + } + + /// + /// Finds the root path for resolving configuration file paths. Assumes that the .git-containing + /// directory is the root path. + /// + protected virtual PathBasedConfigFile CreateConfigFileWithHeuristic( + string configPath, string? commandLineRoot) + { + if (commandLineRoot != null) + { + return new PathBasedConfigFile(configPath, commandLineRoot, identifierPrefix: null); + } + string? parent = Path.GetDirectoryName(configPath); + while (parent != null) + { + if (Directory.Exists(Path.Combine(parent, ".git"))) + { + return new PathBasedConfigFile(configPath, parent, identifierPrefix: null); + } + parent = Path.GetDirectoryName(parent); + } + return new PathBasedConfigFile(configPath, rootPath: null, identifierPrefix: null); + } + + protected virtual Console GetConsole(string[] args) + { + bool verbose = IsVerbose(args); + Console console; + if (System.Console.IsOutputRedirected || System.Console.IsInputRedirected) + { + console = LogConsole.WriteOnlyConsole(System.Console.Error, verbose); + } + else if (args.Contains(GeneralOptions.Noansi)) + { + console = LogConsole.ReadWriteConsole(System.Console.In, System.Console.Error, verbose); + } + else + { + console = new AnsiConsole(System.Console.In, System.Console.Error, verbose); + } + + string? noPrompt = FindFlagValue(args, GeneralOptions.Noprompt); + if (noPrompt == "true") + { + console = new NoPromptConsole(console, true); + } + + string? maybeConsoleFilePath = FindFlagValue(args, ConsoleFilePathFlag); + if (maybeConsoleFilePath == null) + { + return console; + } + + try + { + string? dir = Path.GetDirectoryName(maybeConsoleFilePath); + if (!string.IsNullOrEmpty(dir)) + { + Directory.CreateDirectory(dir); + } + } + catch (IOException) + { + // Could not create parent directories; disable redirecting. + return console; + } + return new FileConsole(console, maybeConsoleFilePath, GetConsoleFlushRate(args)); + } + + /// Returns the console flush rate from the flag, if valid, or the default otherwise. + protected virtual TimeSpan GetConsoleFlushRate(string[] args) + { + string? value = FindFlagValue(args, ConsoleFileFlushIntervalFlag); + return value != null ? DurationConverter.Convert(value) + : DefaultConsoleFileFlushInterval; + } + + /// + /// Hook to allow setting variables that are not run/validation specific, based on options. Called + /// after command-line options are parsed but before a file is read or a run started. + /// + protected virtual void InitEnvironment(Options options, ICopybaraCmd copybaraCmd, string[] rawArgs) + { + GeneralOptions generalOptions = options.Get(); + Profiler = generalOptions.Profiler(); + var profilerListeners = new List + { + new LogProfilerListener(), + new ConsoleProfilerListener(generalOptions.GetConsole()), + }; + Profiler.Init(profilerListeners); + CleanupOutputDir(generalOptions); + } + + protected virtual void CleanupOutputDir(GeneralOptions generalOptions) + { + generalOptions.IoRepoTask( + "clean_outputdir", + () => + { + if (generalOptions.IsNoCleanup()) + { + return null; + } + generalOptions.GetConsole().Progress("Cleaning output directory"); + generalOptions.GetDirFactory().CleanupTempDirs(); + return null; + }); + } + + /// Performs cleanup tasks after executing Copybara. + protected virtual void Shutdown(CommandResult result) + { + if (_console != null) + { + _console.Dispose(); + } + if (Profiler != null) + { + Profiler.Stop(); + } + } + + protected virtual void HandleUnexpectedError( + Console console, string msg, string[] args, Exception e) + { + console.Error(msg + " (" + e + ")"); + } + + internal string Usage() + { + var sb = new System.Text.StringBuilder(); + sb.Append("Copybara version: ").Append(GetVersion()).Append('\n'); + sb.Append("Usage: copybara [subcommand] ").Append(MainArguments.CopybaraSkylarkConfigFilename) + .Append(" [migration_name [source_ref]]\n\n"); + sb.Append("Available subcommands: migrate, info, validate, version, help\n\n"); + if (ArgParser != null) + { + sb.Append("Flags:\n"); + foreach (var (names, description) in ArgParser.Descriptions.OrderBy(d => d.Names, StringComparer.Ordinal)) + { + sb.Append(" ").Append(names).Append("\n ").Append(description).Append('\n'); + } + } + sb.Append("\nExample:\n copybara ").Append(MainArguments.CopybaraSkylarkConfigFilename) + .Append(" origin/main\n"); + return sb.ToString(); + } + + private static IReadOnlyDictionary GetSystemEnvironment() + { + var dict = new Dictionary(StringComparer.Ordinal); + foreach (System.Collections.DictionaryEntry e in System.Environment.GetEnvironmentVariables()) + { + dict[(string)e.Key] = (string?)e.Value ?? ""; + } + return dict; + } + + /// Prints the Copybara version. + private sealed class VersionCmd : ICopybaraCmd + { + private readonly Main _main; + + public VersionCmd(Main main) => _main = main; + + public ExitCode Run(CommandEnv commandEnv) + { + commandEnv.GetOptions().Get().GetConsole().Info(_main.GetBinaryInfo()); + return ExitCode.Success; + } + + public string Name => "version"; + } + + /// Prints the help message. + private sealed class HelpCmd : ICopybaraCmd + { + private readonly Main _main; + + public HelpCmd(Main main) => _main = Preconditions.CheckNotNull(main); + + public ExitCode Run(CommandEnv commandEnv) + { + commandEnv.GetOptions().Get().GetConsole().Info(_main.Usage()); + return ExitCode.Success; + } + + public string Name => "help"; + } +} diff --git a/src/Copybara.Cli/MainArguments.cs b/src/Copybara.Cli/MainArguments.cs new file mode 100644 index 000000000..e291cd335 --- /dev/null +++ b/src/Copybara.Cli/MainArguments.cs @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Util; + +namespace Copybara.Cli; + +/// +/// Arguments which are unnamed (i.e. positional) or must be evaluated inside . +/// +public sealed class MainArguments +{ + public const string CopybaraSkylarkConfigFilename = "copy.bara.sky"; + + private readonly ImmutableArray _rawArgs; + + public MainArguments(IEnumerable rawArgs) + { + _rawArgs = rawArgs.ToImmutableArray(); + } + + /// The positional arguments parsed off the command line (config, migration, refs). + public List Unnamed { get; set; } = new(); + + [Flag( + "--work-dir", + "Directory where all the transformations will be performed. By default a temporary" + + " directory.")] + public string? BaseWorkdir { get; set; } + + public IReadOnlyList GetRawArgs() => _rawArgs; + + /// + /// Returns the base working directory. This method should not be accessed directly by any other + /// class but . + /// + public string GetBaseWorkdir(GeneralOptions generalOptions, string fileSystemRoot) + { + _ = fileSystemRoot; + string workdirPath = BaseWorkdir == null + ? generalOptions.GetDirFactory().NewTempDir("workdir") + : Path.GetFullPath(BaseWorkdir); + + if (File.Exists(workdirPath) && !Directory.Exists(workdirPath)) + { + throw new IOException($"'{workdirPath}' exists and is not a directory"); + } + + if (Directory.Exists(workdirPath) && !IsDirEmpty(workdirPath)) + { + System.Console.Error.WriteLine($"WARNING: {workdirPath} is not empty"); + } + + return workdirPath; + } + + private static bool IsDirEmpty(string directory) => + !Directory.EnumerateFileSystemEntries(directory).Any(); + + /// + /// Resolves the subcommand and its remaining args from the positional arguments, mirroring + /// upstream's config-vs-command disambiguation logic. + /// + /// + public CommandWithArgs ParseCommand( + IReadOnlyDictionary commands, ICopybaraCmd defaultCmd) + { + if (Unnamed.Count == 0) + { + return new CommandWithArgs(defaultCmd, ImmutableArray.Empty); + } + + string firstArg = Unnamed[0]; + // Default command might take a config file as param. + if (firstArg.EndsWith(CopybaraSkylarkConfigFilename, StringComparison.Ordinal)) + { + return new CommandWithArgs(defaultCmd, Unnamed.ToImmutableArray()); + } + + if (firstArg.Contains(CopybaraSkylarkConfigFilename + ':', StringComparison.Ordinal)) + { + var args = ImmutableArray.CreateBuilder(); + args.AddRange(SplitConfigArg(firstArg)); + args.AddRange(Unnamed.Skip(1)); + return new CommandWithArgs(defaultCmd, args.ToImmutable()); + } + + string key = firstArg.ToLowerInvariant(); + if (!commands.ContainsKey(key)) + { + var available = new SortedSet(commands.Keys, StringComparer.Ordinal); + throw new CommandLineException( + $"Invalid subcommand '{firstArg}'. Available commands: [{string.Join(", ", available)}]"); + } + + if (Unnamed.Count == 1) + { + return new CommandWithArgs(commands[key], ImmutableArray.Empty); + } + + var rest = ImmutableArray.CreateBuilder(); + rest.AddRange(SplitConfigArg(Unnamed[1])); + rest.AddRange(Unnamed.Skip(2)); + return new CommandWithArgs(commands[key], rest.ToImmutable()); + } + + private static IReadOnlyList SplitConfigArg(string arg) + { + int idx = arg.IndexOf("copy.bara.sky:", StringComparison.Ordinal); + if (idx < 0) + { + return ImmutableArray.Create(arg); + } + string head = arg.Substring(0, idx) + "copy.bara.sky"; + string tail = arg.Substring(idx + "copy.bara.sky:".Length); + return ImmutableArray.Create(head, tail); + } + + /// A subcommand and the remaining (config/migration/ref) arguments for it. + public sealed class CommandWithArgs + { + internal CommandWithArgs(ICopybaraCmd subcommand, ImmutableArray args) + { + Subcommand = Preconditions.CheckNotNull(subcommand); + Args = args; + } + + public ICopybaraCmd Subcommand { get; } + + public IReadOnlyList Args { get; } + } +} diff --git a/src/Copybara.Cli/MigrateCmd.cs b/src/Copybara.Cli/MigrateCmd.cs new file mode 100644 index 000000000..6f0da0b38 --- /dev/null +++ b/src/Copybara.Cli/MigrateCmd.cs @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara; +using Copybara.Common; +using Copybara.Config; +using Copybara.Exceptions; +using Copybara.Util; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Cli; + +/// +/// Executes the migration for the given config. +/// +public sealed class MigrateCmd : ICopybaraCmd +{ + private readonly ConfigValidator _configValidator; + private readonly Action _migrationRanConsumer; + private readonly IConfigLoaderProvider _configLoaderProvider; + private readonly ModuleSet _moduleSet; + + public MigrateCmd( + ConfigValidator configValidator, + Action migrationRanConsumer, + IConfigLoaderProvider configLoaderProvider, + ModuleSet moduleSet) + { + _configValidator = Preconditions.CheckNotNull(configValidator); + _migrationRanConsumer = Preconditions.CheckNotNull(migrationRanConsumer); + _configLoaderProvider = Preconditions.CheckNotNull(configLoaderProvider); + _moduleSet = moduleSet; + } + + public ExitCode Run(CommandEnv commandEnv) + { + ConfigFileArgs configFileArgs = commandEnv.GetConfigFileArgs()!; + IReadOnlyList sourceRefs = configFileArgs.GetSourceRefs(); + string workflowName = configFileArgs.GetWorkflowName(); + UpdateEnvironment(workflowName); + GeneralOptions generalOptions = commandEnv.GetOptions().Get(); + Console console = generalOptions.GetConsole(); + console.VerboseFmt("Executing workflow '%s'", workflowName); + Run( + commandEnv.GetOptions(), + _configLoaderProvider.NewLoader( + configFileArgs.GetConfigPath(), + sourceRefs.Count == 1 ? sourceRefs[0] : null), + workflowName, + commandEnv.GetWorkdir(), + sourceRefs); + return ExitCode.Success; + } + + /// Runs the migration specified by . + private void Run( + Options options, + ConfigLoader configLoader, + string migrationName, + string workdir, + IReadOnlyList sourceRefs) + { + Config.Config config = LoadConfig(options, configLoader, migrationName); + + IMigration migration = config.GetMigration(migrationName); + + if (!options.Get().IsReadConfigFromChange()) + { + _migrationRanConsumer(migration); + migration.Run(workdir, sourceRefs); + return; + } + + ValidationException.CheckCondition( + configLoader.SupportsLoadForRevision(), + "{0} flag is not supported for the origin/config file path", + "--read-config-from-change"); + + // A safeguard, mirror workflows are not supported in the service anyway. + ValidationException.CheckCondition( + migration is Workflow, + "Flag --read-config-from-change is not supported for non-workflow migrations: {0}", + migrationName); + _migrationRanConsumer(migration); + + // TODO(port): ReadConfigFromChangeWorkflow is not ported yet. When it lands, replace the + // fallthrough below with the equivalent of: + // new ReadConfigFromChangeWorkflow(workflow, options, configLoader, configValidator) + // .run(workdir, sourceRefs); + throw new ValidationException( + "--read-config-from-change is not yet supported in the .NET port."); + } + + private Config.Config LoadConfig(Options options, ConfigLoader configLoader, string migrationName) + { + GeneralOptions generalOptions = options.Get(); + Console console = generalOptions.GetConsole(); + Config.Config config = configLoader.Load(console); + console.Progress("Validating configuration"); + ValidationResult result = _configValidator.Validate(config, migrationName); + if (!result.HasErrors()) + { + return config; + } + + foreach (string error in result.GetErrors()) + { + console.Error(error); + } + console.Error("Configuration is invalid."); + throw new ValidationException( + "Error validating configuration: Configuration is invalid."); + } + + private void UpdateEnvironment(string migrationName) + { + foreach (object module in _moduleSet.GetModules().Values) + { + // We mutate the module per file loaded. Not ideal but it is the best we can do. + if (module is ILabelsAwareModule m) + { + m.SetWorkflowName(migrationName); + } + } + } + + public string Name => "migrate"; +} diff --git a/src/Copybara.Cli/Program.cs b/src/Copybara.Cli/Program.cs new file mode 100644 index 000000000..67eb8b1b1 --- /dev/null +++ b/src/Copybara.Cli/Program.cs @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Entry point for the `copybara` .NET tool. Delegates to Copybara.Cli.Main, which mirrors the +// orchestration flow of the upstream com.google.copybara.Main class. + +using Copybara.Cli; +using Copybara.Util; + +return (int)new Main().Run(args); diff --git a/src/Copybara.Cli/ValidateCmd.cs b/src/Copybara.Cli/ValidateCmd.cs new file mode 100644 index 000000000..a0fcb6920 --- /dev/null +++ b/src/Copybara.Cli/ValidateCmd.cs @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara; +using Copybara.Common; +using Copybara.Config; +using Copybara.Exceptions; +using Copybara.Util; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Cli; + +/// +/// Validates that the configuration is correct. +/// +public sealed class ValidateCmd : ICopybaraCmd +{ + private readonly ConfigValidator _configValidator; + private readonly IConfigLoaderProvider _configLoaderProvider; + + public ValidateCmd( + ConfigValidator configValidator, + Action migrationRanConsumer, + IConfigLoaderProvider configLoaderProvider) + { + _ = migrationRanConsumer; + _configValidator = Preconditions.CheckNotNull(configValidator); + _configLoaderProvider = Preconditions.CheckNotNull(configLoaderProvider); + } + + public ExitCode Run(CommandEnv commandEnv) + { + ConfigFileArgs configFileArgs = commandEnv.GetConfigFileArgs()!; + ConfigLoader configLoader = + _configLoaderProvider.NewLoader( + configFileArgs.GetConfigPath(), configFileArgs.GetSourceRef()); + ValidationResult result = + Validate(commandEnv.GetOptions(), configLoader, configFileArgs.GetWorkflowName()); + + Console console = commandEnv.GetOptions().Get().GetConsole(); + foreach (var message in result.GetAllMessages()) + { + switch (message.GetLevel()) + { + case ValidationResult.Level.WARNING: + console.Warn(message.GetMessage()); + break; + case ValidationResult.Level.ERROR: + console.Error(message.GetMessage()); + break; + } + } + + if (result.HasErrors()) + { + console.ErrorFmt("Configuration '%s' is invalid.", configLoader.Location()); + return ExitCode.ConfigurationError; + } + + console.InfoFmt("Configuration '%s' is valid.", configLoader.Location()); + return ExitCode.Success; + } + + /// + /// Validates that the configuration is correct and that there is a valid migration specified by + /// . + /// + /// Note that, besides validating the specific migration, all the configuration will be + /// validated syntactically. + /// + private ValidationResult Validate(Options options, ConfigLoader configLoader, string migrationName) + { + Console console = options.Get().GetConsole(); + var resultBuilder = new ValidationResult.Builder(); + try + { + Config.Config config = configLoader.Load(console); + resultBuilder.Append(_configValidator.Validate(config, migrationName)); + } + catch (ValidationException e) + { + // The validate subcommand should not throw Validation exceptions but log a result. + var error = new System.Text.StringBuilder(e.Message).Append('\n'); + Exception? cause = e.InnerException; + while (cause != null) + { + error.Append(" CAUSED BY: ").Append(cause.Message).Append('\n'); + cause = cause.InnerException; + } + resultBuilder.Error(error.ToString()); + } + + return resultBuilder.Build(); + } + + public string Name => "validate"; +} diff --git a/src/Copybara.Common/Copybara.Common.csproj b/src/Copybara.Common/Copybara.Common.csproj new file mode 100644 index 000000000..f753f338c --- /dev/null +++ b/src/Copybara.Common/Copybara.Common.csproj @@ -0,0 +1,6 @@ + + + Copybara.Common + Copybara.Common + + diff --git a/src/Copybara.Common/ImmutableListMultimap.cs b/src/Copybara.Common/ImmutableListMultimap.cs new file mode 100644 index 000000000..1941f9c1c --- /dev/null +++ b/src/Copybara.Common/ImmutableListMultimap.cs @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections; +using System.Collections.Immutable; + +namespace Copybara.Common; + +/// +/// An immutable multimap preserving insertion order of keys and values, mirroring the +/// subset of Guava's ImmutableListMultimap that Copybara relies on. A key may map to +/// multiple values; iteration order matches insertion order. +/// +public sealed class ImmutableListMultimap : IEnumerable> + where TKey : notnull +{ + private readonly ImmutableArray> _entries; + private readonly ImmutableDictionary> _index; + + private ImmutableListMultimap( + ImmutableArray> entries, + ImmutableDictionary> index) + { + _entries = entries; + _index = index; + } + + private static readonly ImmutableListMultimap EmptyInstance = + new(ImmutableArray>.Empty, + ImmutableDictionary>.Empty); + + public static ImmutableListMultimap Empty => EmptyInstance; + + /// Creates a multimap with a single key/value entry (Guava parity for of). + public static ImmutableListMultimap Of(TKey key, TValue value) => + CreateBuilder().Put(key, value).Build(); + + /// All values associated with in insertion order (empty if none). + public ImmutableArray this[TKey key] => + _index.TryGetValue(key, out var values) ? values : ImmutableArray.Empty; + + /// All values associated with in insertion order (empty if none). + public ImmutableArray Get(TKey key) => this[key]; + + public bool ContainsKey(TKey key) => _index.ContainsKey(key); + + public bool ContainsEntry(TKey key, TValue value) => + _index.TryGetValue(key, out var values) && values.Contains(value); + + public IEnumerable Keys => _index.Keys; + + public int Count => _entries.Length; + + public bool IsEmpty => _entries.IsEmpty; + + /// Returns the entries as a map from key to its list of values. + public ImmutableDictionary> AsMap() => _index; + + public IEnumerator> GetEnumerator() => + ((IEnumerable>)_entries).GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public static Builder CreateBuilder() => new(); + + /// Builder that accumulates entries preserving insertion order. + public sealed class Builder + { + private readonly ImmutableArray>.Builder _entries = + ImmutableArray.CreateBuilder>(); + + public Builder Put(TKey key, TValue value) + { + _entries.Add(new KeyValuePair(key, value)); + return this; + } + + public Builder PutAll(TKey key, IEnumerable values) + { + foreach (var value in values) + { + _entries.Add(new KeyValuePair(key, value)); + } + return this; + } + + public Builder PutAll(ImmutableListMultimap multimap) + { + foreach (var entry in multimap) + { + _entries.Add(entry); + } + return this; + } + + public ImmutableListMultimap Build() + { + var entries = _entries.ToImmutable(); + var perKey = new Dictionary.Builder>(); + foreach (var entry in entries) + { + if (!perKey.TryGetValue(entry.Key, out var listBuilder)) + { + listBuilder = ImmutableArray.CreateBuilder(); + perKey[entry.Key] = listBuilder; + } + listBuilder.Add(entry.Value); + } + + var indexBuilder = ImmutableDictionary.CreateBuilder>(); + foreach (var kvp in perKey) + { + indexBuilder[kvp.Key] = kvp.Value.ToImmutable(); + } + + return new ImmutableListMultimap(entries, indexBuilder.ToImmutable()); + } + } +} diff --git a/src/Copybara.Common/ImmutableSetMultimap.cs b/src/Copybara.Common/ImmutableSetMultimap.cs new file mode 100644 index 000000000..376ead69f --- /dev/null +++ b/src/Copybara.Common/ImmutableSetMultimap.cs @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections; +using System.Collections.Immutable; + +namespace Copybara.Common; + +/// +/// An immutable multimap where the values for each key form a set (no duplicate values per key), +/// mirroring the subset of Guava's ImmutableSetMultimap that Copybara relies on. Key insertion +/// order is preserved; per key, values keep their first-seen insertion order and duplicates are +/// dropped. +/// +public sealed class ImmutableSetMultimap : IEnumerable> + where TKey : notnull +{ + private readonly ImmutableArray> _entries; + private readonly ImmutableDictionary> _index; + + private ImmutableSetMultimap( + ImmutableArray> entries, + ImmutableDictionary> index) + { + _entries = entries; + _index = index; + } + + private static readonly ImmutableSetMultimap EmptyInstance = + new(ImmutableArray>.Empty, + ImmutableDictionary>.Empty); + + public static ImmutableSetMultimap Empty => EmptyInstance; + + /// All values associated with (empty if none). + public ImmutableHashSet this[TKey key] => + _index.TryGetValue(key, out var values) ? values : ImmutableHashSet.Empty; + + /// All values associated with (empty if none). + public ImmutableHashSet Get(TKey key) => this[key]; + + public bool ContainsKey(TKey key) => _index.ContainsKey(key); + + public bool ContainsEntry(TKey key, TValue value) => + _index.TryGetValue(key, out var values) && values.Contains(value); + + public IEnumerable Keys => _index.Keys; + + public int Count => _entries.Length; + + public bool IsEmpty => _entries.IsEmpty; + + /// Returns the entries as a map from key to its set of values. + public ImmutableDictionary> AsMap() => _index; + + public IEnumerator> GetEnumerator() => + ((IEnumerable>)_entries).GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public static Builder CreateBuilder() => new(); + + /// Builder that accumulates entries, dropping duplicate (key, value) pairs. + public sealed class Builder + { + private readonly List> _entries = new(); + private readonly HashSet<(TKey, TValue)> _seen = new(); + + public Builder Put(TKey key, TValue value) + { + if (_seen.Add((key, value))) + { + _entries.Add(new KeyValuePair(key, value)); + } + return this; + } + + public Builder PutAll(TKey key, IEnumerable values) + { + foreach (var value in values) + { + Put(key, value); + } + return this; + } + + public Builder PutAll(ImmutableSetMultimap multimap) + { + foreach (var entry in multimap) + { + Put(entry.Key, entry.Value); + } + return this; + } + + public ImmutableSetMultimap Build() + { + var entries = _entries.ToImmutableArray(); + var perKey = new Dictionary.Builder>(); + var order = new List(); + foreach (var entry in entries) + { + if (!perKey.TryGetValue(entry.Key, out var setBuilder)) + { + setBuilder = ImmutableHashSet.CreateBuilder(); + perKey[entry.Key] = setBuilder; + order.Add(entry.Key); + } + setBuilder.Add(entry.Value); + } + + var indexBuilder = ImmutableDictionary.CreateBuilder>(); + foreach (var key in order) + { + indexBuilder[key] = perKey[key].ToImmutable(); + } + + return new ImmutableSetMultimap(entries, indexBuilder.ToImmutable()); + } + } +} diff --git a/src/Copybara.Common/Preconditions.cs b/src/Copybara.Common/Preconditions.cs new file mode 100644 index 000000000..fb2b39300 --- /dev/null +++ b/src/Copybara.Common/Preconditions.cs @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Diagnostics.CodeAnalysis; + +namespace Copybara.Common; + +/// +/// Guava-style precondition helpers. Ported from com.google.common.base.Preconditions +/// (only the members Copybara actually uses). +/// +public static class Preconditions +{ + /// Ensures is not null, returning it. + public static T CheckNotNull([NotNull] T? reference) where T : class + { + if (reference is null) + { + throw new ArgumentNullException(); + } + return reference; + } + + /// Ensures is not null, returning it. + public static T CheckNotNull([NotNull] T? reference, object? errorMessage) where T : class + { + if (reference is null) + { + throw new ArgumentNullException(null, errorMessage?.ToString()); + } + return reference; + } + + /// Ensures is not null, returning it. + public static T CheckNotNull([NotNull] T? reference, string format, params object?[] args) + where T : class + { + if (reference is null) + { + throw new ArgumentNullException(null, string.Format(format, args)); + } + return reference; + } + + /// Ensures a nullable value type is not null, returning its value. + public static T CheckNotNull([NotNull] T? reference) where T : struct + { + if (reference is null) + { + throw new ArgumentNullException(); + } + return reference.Value; + } + + /// Ensures a nullable value type is not null, returning its value. + public static T CheckNotNull([NotNull] T? reference, object? errorMessage) where T : struct + { + if (reference is null) + { + throw new ArgumentNullException(null, errorMessage?.ToString()); + } + return reference.Value; + } + + /// Ensures a nullable value type is not null, returning its value. + public static T CheckNotNull([NotNull] T? reference, string format, params object?[] args) + where T : struct + { + if (reference is null) + { + throw new ArgumentNullException(null, string.Format(format, args)); + } + return reference.Value; + } + + /// Ensures the truth of an expression involving parameters to the calling method. + public static void CheckArgument([DoesNotReturnIf(false)] bool expression) + { + if (!expression) + { + throw new ArgumentException(); + } + } + + /// Ensures the truth of an expression involving parameters to the calling method. + public static void CheckArgument([DoesNotReturnIf(false)] bool expression, object? errorMessage) + { + if (!expression) + { + throw new ArgumentException(errorMessage?.ToString()); + } + } + + /// Ensures the truth of an expression involving parameters to the calling method. + public static void CheckArgument( + [DoesNotReturnIf(false)] bool expression, string format, params object?[] args) + { + if (!expression) + { + throw new ArgumentException(string.Format(format, args)); + } + } + + /// Ensures the truth of an expression involving the state of the calling instance. + public static void CheckState([DoesNotReturnIf(false)] bool expression) + { + if (!expression) + { + throw new InvalidOperationException(); + } + } + + /// Ensures the truth of an expression involving the state of the calling instance. + public static void CheckState([DoesNotReturnIf(false)] bool expression, object? errorMessage) + { + if (!expression) + { + throw new InvalidOperationException(errorMessage?.ToString()); + } + } + + /// Ensures the truth of an expression involving the state of the calling instance. + public static void CheckState( + [DoesNotReturnIf(false)] bool expression, string format, params object?[] args) + { + if (!expression) + { + throw new InvalidOperationException(string.Format(format, args)); + } + } +} diff --git a/src/Copybara.Core/Action/ActionContext.cs b/src/Copybara.Core/Action/ActionContext.cs new file mode 100644 index 000000000..b660507a8 --- /dev/null +++ b/src/Copybara.Core/Action/ActionContext.cs @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2021 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Config; +using Copybara.Effect; +using Copybara.Exceptions; +using Copybara.Revision; +using Copybara.Transform; +using Starlark.Annot; +using Starlark.Eval; +using ConsoleT = Copybara.Util.Console.Console; +using StarlarkRt = Starlark.Eval.Starlark; +using Sequence = Starlark.Eval.Sequence; + +namespace Copybara.Action; + +/// +/// A StarlarkContext for running Actions. Port of com.google.copybara.action.ActionContext. +/// +public abstract class ActionContext : ISkylarkContext, IStarlarkValue + where T : ISkylarkContext +{ + protected readonly List NewDestinationEffects = new(); + private ActionResult? _actionResult; + + protected ActionContext( + IAction action, + SkylarkConsole console, + IReadOnlyDictionary labels, + Dict @params) + { + Action = Preconditions.CheckNotNull(action); + Console = Preconditions.CheckNotNull(console); + Labels = Preconditions.CheckNotNull(labels); + Params = Preconditions.CheckNotNull(@params); + } + + /// The action this context runs. + protected IAction Action { get; } + + /// The console used to report errors or warnings. + protected SkylarkConsole Console { get; } + + /// The CLI labels passed to the migration. + protected IReadOnlyDictionary Labels { get; } + + private Dict Params { get; } + + [StarlarkMethod( + "action_name", + Doc = "The name of the current action.", + StructField = true)] + public string GetActionName() => Action.GetName(); + + [StarlarkMethod( + "console", + Doc = "Get an instance of the console to report errors or warnings", + StructField = true)] + public ConsoleT GetConsole() => Console; + + [StarlarkMethod( + "params", + Doc = "Parameters for the function if created with core.action", + StructField = true)] + public Dict GetParams() => Params; + + [StarlarkMethod("success", Doc = "Returns a successful action result.")] + public ActionResult Success() => ActionResult.SuccessResult(); + + [StarlarkMethod( + "noop", + Doc = "Returns a no op action result with an optional message.")] + public ActionResult Noop( + [Param( + Name = "msg", + AllowedTypes = new[] { typeof(string), typeof(NoneType) }, + Doc = "The no op message", + DefaultValue = "None")] + object? noopMsg) => + ActionResult.NoopResult(SkylarkUtil.ConvertFromNoneable(noopMsg, null)); + + [StarlarkMethod( + "error", + Doc = "Returns an error action result.")] + public ActionResult Error( + [Param(Name = "msg", Doc = "The error message")] string errorMsg) => + ActionResult.ErrorResult(errorMsg); + + /// Return the new s created by this context. + public IReadOnlyList GetNewDestinationEffects() => + NewDestinationEffects.ToImmutableArray(); + + [StarlarkMethod( + "cli_labels", + Doc = "Access labels that a user passes through flag '--labels'. " + + "For example: --labels=foo:value1,bar:value2. Then it can access in this way:" + + "cli_labels['foo'].", + StructField = true)] + public Dict GetCliLabels() => + Dict.ImmutableCopyOf( + Labels.Select(kv => new KeyValuePair(kv.Key, kv.Value))); + + [StarlarkMethod( + "record_effect", + Doc = "Records an effect of the current action.")] + public void RecordEffect( + [Param(Name = "summary", Doc = "The summary of this effect", Named = true)] + string summary, + [Param( + Name = "origin_refs", + AllowedTypes = new[] { typeof(ISequence) }, + Doc = "The origin refs", + Named = true)] + ISequence originRefs, + [Param(Name = "destination_ref", Doc = "The destination ref", Named = true)] + DestinationEffect.DestinationRef destinationRef, + [Param( + Name = "errors", + AllowedTypes = new[] { typeof(ISequence) }, + DefaultValue = "[]", + Doc = "An optional list of errors", + Named = true)] + ISequence errors, + [Param( + Name = "type", + Doc = "The type of migration effect (CREATED, UPDATED, NOOP," + + " NOOP_AGAINST_PENDING_CHANGE, INSUFFICIENT_APPROVALS, ERROR, STARTED).", + DefaultValue = "\"UPDATED\"", + Named = true)] + string typeStr) + { + var type = SkylarkUtil.StringToEnum("type", typeStr); + NewDestinationEffects.Add( + new DestinationEffect( + type, + summary, + Sequence.Cast(originRefs, "origin_refs"), + destinationRef, + Sequence.Cast(errors, "errors"))); + } + + public abstract T WithParams(Dict @params); + + public void OnFinish(object? result, object context) + { + ValidationException.CheckCondition( + result != null, + "Actions must return a result via built-in functions: success(), " + + "error(), noop() return, but '{0}' returned: None", Action.GetName()); + ValidationException.CheckCondition( + result is ActionResult, + "Actions must return a result via built-in functions: success(), " + + "error(), noop() return, but '{0}' returned: {1}", Action.GetName(), result!); + _actionResult = (ActionResult)result!; + switch (_actionResult.GetResult()) + { + case ActionResult.Result.Error: + Console.ErrorFmt( + "Action '{0}' returned error: {1}", Action.GetName(), _actionResult.GetMsg()!); + break; + case ActionResult.Result.NoOp: + Console.InfoFmt( + "Action '{0}' returned noop: {1}", Action.GetName(), _actionResult.GetMsg()!); + break; + case ActionResult.Result.Success: + Console.InfoFmt("Action '{0}' returned success", Action.GetName()); + break; + } + + // Populate effects registered in the action context. This is required because StarlarkAction + // makes a copy of the context to inject the parameters, but that instance is not visible from + // the caller. + if (context is ActionContext other) + { + NewDestinationEffects.AddRange(other.NewDestinationEffects); + } + } + + public ActionResult? GetActionResult() => _actionResult; +} diff --git a/src/Copybara.Core/Action/ActionResult.cs b/src/Copybara.Core/Action/ActionResult.cs new file mode 100644 index 000000000..a3d9a323c --- /dev/null +++ b/src/Copybara.Core/Action/ActionResult.cs @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Action; + +/// +/// Represents the result returned by an . Port of +/// com.google.copybara.action.ActionResult. +/// +[StarlarkBuiltin( + "dynamic.action_result", + Doc = "Result objects created by actions to tell Copybara what happened.")] +public sealed class ActionResult : IStarlarkPrintableValue +{ + private readonly Result _result; + private readonly string? _msg; + + private ActionResult(Result result, string? msg) + { + _result = result; + _msg = msg; + } + + /// Result kind. The Starlark-facing name is the upstream-compatible upper snake case form. + public enum Result + { + Success, + Error, + NoOp, + } + + public static ActionResult SuccessResult() => new(Result.Success, msg: null); + + public static ActionResult ErrorResult(string msg) => new(Result.Error, msg); + + public static ActionResult NoopResult(string? msg) => new(Result.NoOp, msg); + + public Result GetResult() => _result; + + [StarlarkMethod( + "result", + Doc = "The result of this action", + StructField = true)] + public string GetResultForSkylark() => ResultName(_result); + + [StarlarkMethod( + "msg", + Doc = "The message associated with the result", + StructField = true, + AllowReturnNones = true)] + public string? GetMsg() => _msg; + + /// Upstream-compatible name of the result (SUCCESS, ERROR, NO_OP). + private static string ResultName(Result result) => result switch + { + Result.Success => "SUCCESS", + Result.Error => "ERROR", + Result.NoOp => "NO_OP", + _ => result.ToString(), + }; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"ActionResult{{result={ResultName(_result)}, msg={_msg ?? "null"}}}"; +} diff --git a/src/Copybara.Core/Action/IAction.cs b/src/Copybara.Core/Action/IAction.cs new file mode 100644 index 000000000..bb276fb69 --- /dev/null +++ b/src/Copybara.Core/Action/IAction.cs @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Action; + +/// +/// Actions are Starlark functions that receive a context object (that is different depending on where +/// it is used) that expose an API to implement custom logic in Starlark. Port of +/// com.google.copybara.action.Action. +/// +[StarlarkBuiltin( + "dynamic.action", + Doc = + "An action is an Starlark piece of code that does part of a migration. It is used" + + "to define the logic of migration for feedback workflow, on_finish hooks, git.mirror," + + " etc.", + Documented = false)] +public interface IAction : IStarlarkValue +{ + /// + /// Runs the action against the given context. + /// + /// if failure is attributable to user setup. + /// if access to the origin/destination fails. + void Run(ActionContext context) + where T : ISkylarkContext; + + string GetName(); + + /// Returns a key-value list of the options the action was instantiated with. + ImmutableListMultimap Describe(); +} diff --git a/src/Copybara.Core/Action/StarlarkAction.cs b/src/Copybara.Core/Action/StarlarkAction.cs new file mode 100644 index 000000000..60af40ab5 --- /dev/null +++ b/src/Copybara.Core/Action/StarlarkAction.cs @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Copybara.Exceptions; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara.Action; + +/// +/// An implementation of that delegates to a Starlark function. Port of +/// com.google.copybara.action.StarlarkAction. +/// +public sealed class StarlarkAction : IAction +{ + private readonly string _name; + private readonly IStarlarkCallable _function; + private readonly Dict _params; + private readonly StarlarkThread.PrintHandler _printHandler; + + public StarlarkAction( + string name, + IStarlarkCallable function, + Dict @params, + StarlarkThread.PrintHandler printHandler) + { + _name = name; + _function = Preconditions.CheckNotNull(function); + _params = Preconditions.CheckNotNull(@params); + _printHandler = Preconditions.CheckNotNull(printHandler); + } + + public void Run(ActionContext context) + where T : ISkylarkContext + { + T actionContext = context.WithParams(_params); + using var mu = Mutability.Create("dynamic_action"); + try + { + var thread = StarlarkThread.CreateTransient(mu, StarlarkSemantics.DEFAULT); + thread.SetPrintHandler(_printHandler); + object? result = StarlarkRt.Fastcall( + thread, _function, new object?[] { actionContext }, Array.Empty()); + context.OnFinish(result, actionContext); + } + catch (EvalException e) + { + var cause = e.InnerException; + string error = string.Format( + "Error while executing the skylark transformation {0}: {1}.", + _function.Name, e.Message); + if (cause is RepoException) + { + throw new RepoException(error, cause); + } + throw new ValidationException(error, cause); + } + } + + public string GetName() => _name; + + public ImmutableListMultimap Describe() + { + var builder = ImmutableListMultimap.CreateBuilder(); + foreach (var paramKey in _params.Keys) + { + builder.Put( + paramKey?.ToString() ?? "null", + (paramKey is null ? null : _params.Get(paramKey))?.ToString() ?? "null"); + } + return builder.Build(); + } + + public override string ToString() => $"StarlarkAction{{name={_function.Name}}}"; +} diff --git a/src/Copybara.Core/ActionMigration.cs b/src/Copybara.Core/ActionMigration.cs new file mode 100644 index 000000000..0649f0531 --- /dev/null +++ b/src/Copybara.Core/ActionMigration.cs @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Action; +using Copybara.Common; +using Copybara.Config; +using Copybara.Effect; +using Copybara.Exceptions; +using Copybara.Monitor; +using Copybara.Transform; +using Starlark.Eval; + +namespace Copybara; + +/// A migration that can move code or metadata between endpoints. +public class ActionMigration : IMigration +{ + public const string DestinationEndpointName = "destination"; + + private readonly string _name; + private readonly string? _description; + private readonly ConfigFile _configFile; + private readonly ITrigger _trigger; + private readonly IStructure _endpoints; + private readonly IReadOnlyList _actions; + private readonly GeneralOptions _generalOptions; + private readonly string _mode; + private readonly bool _fileSystem; + private readonly ImmutableArray _definitionStack; + + public ActionMigration( + string name, + string? description, + ConfigFile configFile, + ITrigger trigger, + IStructure endpoints, + IReadOnlyList actions, + GeneralOptions generalOptions, + string mode, + bool fileSystem, + ImmutableArray definitionStack) + { + _name = Preconditions.CheckNotNull(name); + _description = description; + _configFile = Preconditions.CheckNotNull(configFile); + _trigger = Preconditions.CheckNotNull(trigger); + _endpoints = endpoints; + _actions = Preconditions.CheckNotNull(actions); + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _mode = mode; + _fileSystem = fileSystem; + _definitionStack = definitionStack; + } + + public void Run(string workdir, IReadOnlyList sourceRefs) + { + var allResultsBuilder = ImmutableArray.CreateBuilder(); + string suffix = string.Join('_', sourceRefs) + .Replace('/', '_') + .Replace(' ', '_'); + string root = "run/" + _name + "/" + suffix.Substring(0, Math.Min(suffix.Length, 20)); + using (Profiler().Start(root)) + { + foreach (var action in _actions) + { + var effects = new List(); + using (Profiler().Start(action.GetName())) + { + try + { + var console = new SkylarkConsole(_generalOptions.GetConsole()); + EventMonitors().DispatchEvent( + m => m.OnChangeMigrationStarted(new IEventMonitor.ChangeMigrationStartedEvent())); + var context = new ActionMigrationContext( + this, action, _generalOptions.CliLabels(), sourceRefs, console); + if (_fileSystem) + { + context = context.WithFileSystem(workdir); + } + action.Run(context); + effects.AddRange(context.GetNewDestinationEffects()); + var actionResult = context.GetActionResult(); + allResultsBuilder.Add(actionResult); + // First error aborts the execution of the other actions. + ValidationException.CheckCondition( + actionResult.GetResult() != ActionResult.Result.Error, + "{0} migration '{1}' action '{2}' returned error: {3}. Aborting execution.", + Capitalize(_mode), _name, action.GetName(), actionResult.GetMsg()); + } + finally + { + EventMonitors().DispatchEvent(m => m.OnChangeMigrationFinished( + new IEventMonitor.ChangeMigrationFinishedEvent( + effects.ToImmutableArray(), + GetOriginDescription(), + GetDestinationDescription()))); + } + } + } + } + var allResults = allResultsBuilder.ToImmutable(); + // This check also returns true if there are no actions. + if (allResults.All(a => a.GetResult() == ActionResult.Result.NoOp)) + { + string detailedMessage = allResults.Length == 0 + ? "actions field is empty" + : "[" + string.Join(", ", allResults.Select(a => a.GetMsg()).Where(m => m != null)) + "]"; + throw new EmptyChangeException( + $"{Capitalize(_mode)} migration '{_name}' was noop. Detailed messages: {detailedMessage}"); + } + } + + private static string Capitalize(string str) => + str.Substring(0, 1).ToUpperInvariant() + str.Substring(1); + + public string GetName() => _name; + + public string? GetDescription() => _description; + + public string GetModeString() => _mode; + + public ConfigFile GetMainConfigFile() => _configFile; + + public ImmutableListMultimap GetOriginDescription() => _trigger.Describe(); + + public ImmutableListMultimap GetDestinationDescription() + { + // We currently require one endpoint to be the designated destination, all others should be + // read only. + object destination = + Preconditions.CheckNotNull(_endpoints.GetValue(DestinationEndpointName)); + return ((IEndpoint)destination).Describe(); + } + + public IReadOnlyDictionary> GetEndpointDescriptions() + { + var result = ImmutableDictionary.CreateBuilder>(); + foreach (var name in _endpoints.GetFieldNames()) + { + if (name != DestinationEndpointName && _endpoints.GetValue(name) is IEndpoint e) + { + result[name] = e.Describe(); + } + } + return result.ToImmutable(); + } + + /// + /// Returns a multimap containing enough data to fingerprint the actions for validation purposes. + /// + public ImmutableListMultimap> GetActionsDescription() + { + var descriptionBuilder = + ImmutableListMultimap>.CreateBuilder(); + foreach (var action in _actions) + { + descriptionBuilder.Put(action.GetName(), action.Describe()); + } + return descriptionBuilder.Build(); + } + + public IReadOnlyList> GetCredentialDescription() + { + var allCreds = ImmutableArray.CreateBuilder>(); + allCreds.AddRange(_trigger.GetEndpoint().DescribeCredentials("trigger")); + foreach (var name in _endpoints.GetFieldNames()) + { + if (_endpoints.GetValue(name) is IEndpoint e) + { + allCreds.AddRange(e.DescribeCredentials(name)); + } + } + return allCreds.ToImmutable(); + } + + internal ITrigger GetTrigger() => _trigger; + + public IStructure GetEndpoints() => _endpoints; + + public override string ToString() => + $"ActionMigration{{name={_name}, trigger={_trigger}, endpoints={_endpoints}," + + $" actions=[{string.Join(", ", _actions)}]}}"; + + private Profiler.Profiler Profiler() => _generalOptions.Profiler(); + + private IEventMonitor.EventMonitors EventMonitors() => _generalOptions.EventMonitors(); + + public ImmutableArray GetDefinitionStack() => _definitionStack; +} diff --git a/src/Copybara.Core/ActionMigrationContext.cs b/src/Copybara.Core/ActionMigrationContext.cs new file mode 100644 index 000000000..787b36b12 --- /dev/null +++ b/src/Copybara.Core/ActionMigrationContext.cs @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Action; +using Copybara.Common; +using Copybara.Transform; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara; + +/// Skylark context for migrations that can do arbitrary endpoint calls and file manipulations. +[StarlarkBuiltin( + // TODO(b/269526710): Rename this. Update docs + "feedback.context", + Doc = + "Gives access to the feedback migration information and utilities. This context is a " + + "concrete implementation for feedback migrations.")] +public class ActionMigrationContext : ActionContext +{ + private readonly ActionMigration _actionMigration; + private readonly ImmutableArray _refs; + private readonly ActionFileSystem? _fs; + + internal ActionMigrationContext( + ActionMigration actionMigration, + IAction currentAction, + IReadOnlyDictionary labels, + IReadOnlyList refs, + SkylarkConsole console) + : this(actionMigration, currentAction, labels, refs, console, Dict.Empty(), fs: null) + { + } + + private ActionMigrationContext( + ActionMigration actionMigration, + IAction currentAction, + IReadOnlyDictionary labels, + IReadOnlyList refs, + SkylarkConsole console, + Dict @params, + ActionFileSystem? fs) + : base(currentAction, console, labels, @params) + { + _actionMigration = Preconditions.CheckNotNull(actionMigration); + _refs = refs.ToImmutableArray(); + _fs = fs; + } + + [StarlarkMethod( + "origin", + Doc = "An object representing the origin. Can be used to query about the ref or modifying" + + " the origin state", + StructField = true)] + public IEndpoint GetOrigin() => + _actionMigration.GetTrigger().GetEndpoint().WithConsole(GetConsole()); + + // TODO(b/269526710): Deprecate this function and use endpoints instead. + [StarlarkMethod( + "destination", + Doc = "An object representing the destination. Can be used to query or modify the" + + " destination state", + StructField = true)] + public IEndpoint GetDestination() + { + if (_actionMigration.GetEndpoints().GetValue("destination") is IEndpoint e) + { + return e; + } + throw new InvalidOperationException("Expected an endpoint called destination"); + } + + [StarlarkMethod( + "endpoints", + Doc = "An object that gives access to the API of the configured endpoints", + StructField = true, + Documented = false)] + public IStructure GetEndpoints() => _actionMigration.GetEndpoints(); + + // TODO(b/269526710): Deprecate this + [StarlarkMethod( + "feedback_name", + Doc = "DEPRECATED: The name of the Feedback migration calling this action." + + " Use migration_name instead.", + StructField = true)] + public string GetFeedbackName() => _actionMigration.GetName(); + + [StarlarkMethod( + "migration_name", + Doc = "The name of the migration calling this action.", + Documented = false, + StructField = true)] + public string GetMigrationName() => _actionMigration.GetName(); + + [StarlarkMethod( + "refs", + Doc = "A list containing string representations of the entities that triggered the event", + StructField = true)] + public StarlarkList GetRefs() => StarlarkList.ImmutableCopyOf(_refs.Cast()); + + [StarlarkMethod( + "fs", + Doc = "If a migration of type `core.action_migration` sets `filesystem = True`, it gives" + + " access to the underlying migration filesystem to manipulate files.", + Documented = false, + StructField = true)] + public ActionFileSystem GetFs() + { + if (_fs == null) + { + throw StarlarkRt.Errorf( + "Migration '{0}' doesn't have access to the filesystem. Use filesystem = True to" + + " enable it", + GetMigrationName()); + } + return _fs; + } + + public override ActionMigrationContext WithParams(Dict @params) => + new(_actionMigration, Action, Labels, _refs, Console, @params, _fs); + + public ActionMigrationContext WithFileSystem(string checkoutDir) => + new(_actionMigration, Action, Labels, _refs, Console, GetParams(), + new ActionFileSystem(checkoutDir)); + + [StarlarkBuiltin( + "action.filesystem", + Doc = "This object gives access to actions to the filesystem for manipulating files.", + Documented = false)] + public sealed class ActionFileSystem : CheckoutFileSystem + { + public ActionFileSystem(string checkoutDir) + : base(checkoutDir) + { + } + } +} diff --git a/src/Copybara.Core/Approval/ChangeWithApprovals.cs b/src/Copybara.Core/Approval/ChangeWithApprovals.cs new file mode 100644 index 000000000..aca88e83a --- /dev/null +++ b/src/Copybara.Core/Approval/ChangeWithApprovals.cs @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Revision; + +namespace Copybara.Approval; + +/// +/// Approvals for a change reference. Port of +/// com.google.copybara.approval.ChangeWithApprovals. +/// TODO: Rename this to Statement +/// (https://github.com/in-toto/attestation/tree/v0.1.0/spec#predicate). +/// +public sealed class ChangeWithApprovals +{ + private readonly Change _change; + private readonly ImmutableArray _predicates; + + public ChangeWithApprovals(Change change) + : this(change, ImmutableArray.Empty) + { + } + + public ChangeWithApprovals( + Change change, ImmutableArray predicates) + { + _change = Preconditions.CheckNotNull(change); + _predicates = predicates; + } + + public Change GetChange() => _change; + + public IReadOnlyList GetPredicates() => _predicates; + + public ChangeWithApprovals AddApprovals(IEnumerable approvals) => + new(_change, _predicates.AddRange(approvals)); + + public override string ToString() => + $"ChangeWithApprovals{{change={_change}, predicates=[{string.Join(", ", _predicates)}]}}"; + + public override bool Equals(object? o) + { + if (ReferenceEquals(this, o)) + { + return true; + } + if (o is not ChangeWithApprovals that) + { + return false; + } + return Equals(_change, that._change) && _predicates.SequenceEqual(that._predicates); + } + + public override int GetHashCode() + { + var hash = new HashCode(); + hash.Add(_change); + foreach (var p in _predicates) + { + hash.Add(p); + } + return hash.ToHashCode(); + } +} diff --git a/src/Copybara.Core/Approval/IApprovalsProvider.cs b/src/Copybara.Core/Approval/IApprovalsProvider.cs new file mode 100644 index 000000000..1a25e5c10 --- /dev/null +++ b/src/Copybara.Core/Approval/IApprovalsProvider.cs @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using ConsoleT = Copybara.Util.Console.Console; + +namespace Copybara.Approval; + +/// +/// An approvals validator that is provided by the origin. Port of +/// com.google.copybara.approval.ApprovalsProvider. +/// +public interface IApprovalsProvider +{ + /// + /// Given a list of changes, return a list of changes that have approvals. + /// + /// changes to be verified with the existing approvals. + /// + /// describes how to find label inputs in case the labels can't be found among + /// . + /// + /// console, in case some message need to be printed. + /// + /// if access to the origin system fails because of being unavailable, server error, etc. + /// + /// + /// if failure is attributable to the user setup (e.g. permission errors, etc.). + /// + ApprovalsResult ComputeApprovals( + ImmutableArray changes, + Func>? labelFinder, + ConsoleT console); +} + +/// +/// An object containing the approvals found for a set of changes. +/// +/// The purpose of this class is to make it easier to migrate to attestations in the future. For +/// example, storing general information about the source. +/// +public sealed class ApprovalsResult +{ + private readonly ImmutableArray _changes; + + public ApprovalsResult(ImmutableArray changes) + { + _changes = changes; + } + + /// + /// List of changes with their corresponding approvals. Must be the complete list of changes, with + /// or without any approval. + /// + public IReadOnlyList GetChanges() => _changes; +} diff --git a/src/Copybara.Core/Approval/NoneApprovedProvider.cs b/src/Copybara.Core/Approval/NoneApprovedProvider.cs new file mode 100644 index 000000000..0bc1f8ab0 --- /dev/null +++ b/src/Copybara.Core/Approval/NoneApprovedProvider.cs @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using ConsoleT = Copybara.Util.Console.Console; + +namespace Copybara.Approval; + +/// +/// An approval provider that returns all the changes as not approved. Port of +/// com.google.copybara.approval.NoneApprovedProvider. +/// +public sealed class NoneApprovedProvider : IApprovalsProvider +{ + public ApprovalsResult ComputeApprovals( + ImmutableArray changes, + Func>? labelFinder, + ConsoleT console) => + new(changes); +} diff --git a/src/Copybara.Core/Approval/StatementPredicate.cs b/src/Copybara.Core/Approval/StatementPredicate.cs new file mode 100644 index 000000000..adb11435f --- /dev/null +++ b/src/Copybara.Core/Approval/StatementPredicate.cs @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; + +namespace Copybara.Approval; + +/// +/// A predicate represents a statement over a change. This is an approximation to +/// predicates. +/// Port of com.google.copybara.approval.StatementPredicate. +/// +public class StatementPredicate +{ + private readonly string _type; + private readonly string _description; + private readonly string _url; + + public StatementPredicate(string type, string description, string url) + { + _type = type; + _description = description; + _url = url; + } + + /// + /// Utility method that filters out elements of that are an instance of + /// . + /// + public static ImmutableArray FilterByClass( + IEnumerable list) + where T : StatementPredicate => + list.OfType().ToImmutableArray(); + + /// Predicate type: Approval, ownership, etc. + public string Type() => _type; + + /// Text representation of the predicate for human consumption. + public string Description() => _description; + + /// + /// Returns where the predicate happened (E.g. an approval in GitHub would have + /// https://github.com/example/project/pull/123 as the url). + /// + public string Url() => _url; + + public override bool Equals(object? o) + { + if (ReferenceEquals(this, o)) + { + return true; + } + if (o is not StatementPredicate that) + { + return false; + } + return string.Equals(_type, that._type) + && string.Equals(_description, that._description) + && string.Equals(_url, that._url); + } + + public override int GetHashCode() => HashCode.Combine(_type, _description, _url); + + public sealed override string ToString() => ToStringDescription(); + + protected virtual string ToStringDescription() => + $"StatementPredicate{{type={_type}, description={_description}, url={_url}}}"; +} diff --git a/src/Copybara.Core/Approval/UserPredicate.cs b/src/Copybara.Core/Approval/UserPredicate.cs new file mode 100644 index 000000000..a4a776aaf --- /dev/null +++ b/src/Copybara.Core/Approval/UserPredicate.cs @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Approval; + +/// +/// Defines a predicate over a user action. Port of +/// com.google.copybara.approval.UserPredicate. +/// +public class UserPredicate : StatementPredicate +{ + private readonly string _username; + private readonly UserPredicateType _type; + + public UserPredicate( + string username, UserPredicateType userType, string originUrl, string description) + : base(userType.ToString(), description, originUrl) + { + _username = username; + _type = userType; + } + + /// + /// String representing the username for the user predicate (e.g. the username of the approver, + /// owners, etc.). + /// + public string Username() => _username; + + public UserPredicateType UserType() => _type; + + /// Type of user predicate. + public enum UserPredicateType + { + /// username is the owner of the change. + OWNER, + + /// + /// username has approved the change. Called LGTM for historical reasons (used internally). + /// + LGTM, + } + + public override bool Equals(object? o) + { + if (ReferenceEquals(this, o)) + { + return true; + } + if (o is not UserPredicate that) + { + return false; + } + if (!base.Equals(o)) + { + return false; + } + return string.Equals(_username, that._username) && _type == that._type; + } + + public override int GetHashCode() => HashCode.Combine(base.GetHashCode(), _username, _type); + + protected override string ToStringDescription() => + base.ToStringDescription() + $" username={_username}, type={_type}"; +} diff --git a/src/Copybara.Core/Archive/ArchiveModule.cs b/src/Copybara.Core/Archive/ArchiveModule.cs new file mode 100644 index 000000000..532849908 --- /dev/null +++ b/src/Copybara.Core/Archive/ArchiveModule.cs @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Config; +using Copybara.Exceptions; +using Copybara.RemoteFile.Extract; +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara.Archive; + +/// A module for handling archives in Starlark. +[StarlarkBuiltin("archive", Doc = "Functions to work with archives.")] +public class ArchiveModule : IStarlarkValue +{ + public ArchiveModule() + { + } + + [StarlarkMethod( + "create", + Doc = "Creates an archive, possibly compressed, from a list of files.")] + public void CreateArchive( + [Param(Name = "archive", Doc = "Expected path of the generated archive file.", Named = true)] + CheckoutPath archivePath, + [Param( + Name = "files", + Doc = + "An optional glob to describe the list of file paths that are to be included in the" + + " archive. If not specified, all files under the current working directory" + + " will be included. Note, the original file path in the filesystem will be" + + " preserved when archiving it.", + Named = true, + DefaultValue = "None", + AllowedTypes = new[] { typeof(Glob), typeof(NoneType) })] + object files) + { + ExtractType type = ResolveArchiveType(archivePath); + try + { + using Stream os = File.Create(archivePath.FullPath()); + ArchiveUtil.CreateArchive( + os, type, archivePath, SkylarkUtil.ConvertFromNoneable(files, null)); + } + catch (Exception e) when (e is IOException or ValidationException) + { + throw StarlarkRt.Errorf("There was an error creating the archive: {0}", e.ToString()); + } + } + + [StarlarkMethod( + "extract", + Doc = "Extract the contents of the archive to a path.")] + public void Extract( + [Param(Name = "archive", Named = true, Doc = "The path to the archive file.")] + CheckoutPath archivePath, + [Param( + Name = "type", + Named = true, + Doc = + "The archive type. Supported types: AUTO, JAR, ZIP, TAR, TAR_GZ, TAR_XZ, and" + + " TAR_BZ2. AUTO will try to infer the archive type automatically.", + DefaultValue = "\"AUTO\"")] + string typeStr, + [Param( + Name = "destination_folder", + Named = true, + Doc = + "The path to extract the archive to. This defaults to the directory where the" + + " archive is located.", + DefaultValue = "None", + AllowedTypes = new[] { typeof(CheckoutPath), typeof(NoneType) })] + object maybeDestination, + [Param( + Name = "paths", + Named = true, + Doc = "An optional glob that is used to filter the files extracted from the archive.", + DefaultValue = "None", + AllowedTypes = new[] { typeof(Glob), typeof(NoneType) })] + object paths) + { + ExtractType type = typeStr.Equals("AUTO") + ? ResolveArchiveType(archivePath) + : SkylarkUtil.StringToEnum("type", typeStr); + + CheckoutPath destination = + SkylarkUtil.ConvertFromNoneable(maybeDestination, archivePath.Resolve(".."))!; + + try + { + using Stream contents = File.OpenRead(archivePath.FullPath()); + ExtractUtil.ExtractArchive( + contents, + destination.FullPath(), + type, + SkylarkUtil.ConvertFromNoneable(paths, null)); + } + catch (Exception e) when (e is IOException or ValidationException) + { + throw StarlarkRt.Errorf("There was an error extracting the archive: {0}", e.ToString()); + } + } + + private static ExtractType ResolveArchiveType(CheckoutPath archivePath) + { + string filename = PathOps.GetFileName(archivePath.GetPath()); + string extension = GetFileExtension(filename); + switch (extension) + { + case "zip": + return ExtractType.ZIP; + case "jar": + return ExtractType.JAR; + case "tar": + return ExtractType.TAR; + case "tgz": + return ExtractType.TAR_GZ; + case "gz": + if (filename.EndsWith(".tar.gz")) + { + return ExtractType.TAR_GZ; + } + goto default; + case "xz": + if (filename.EndsWith(".tar.xz")) + { + return ExtractType.TAR_XZ; + } + goto default; + case "bz2": + if (filename.EndsWith(".tar.bz2")) + { + return ExtractType.TAR_BZ2; + } + goto default; + default: + throw StarlarkRt.Errorf( + "The archive type couldn't be inferred for the file: {0}", + archivePath.GetPath()); + } + } + + /// + /// Mirrors Guava's Files.getFileExtension: returns the substring after the last '.' in + /// the file name, or the empty string if there is no dot. + /// + private static string GetFileExtension(string fileName) + { + int dotIndex = fileName.LastIndexOf('.'); + return dotIndex == -1 ? "" : fileName[(dotIndex + 1)..]; + } +} diff --git a/src/Copybara.Core/Archive/ArchiveUtil.cs b/src/Copybara.Core/Archive/ArchiveUtil.cs new file mode 100644 index 000000000..5557e382b --- /dev/null +++ b/src/Copybara.Core/Archive/ArchiveUtil.cs @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Formats.Tar; +using System.IO.Compression; +using Copybara.Exceptions; +using Copybara.RemoteFile.Extract; +using Copybara.Util; + +namespace Copybara.Archive; + +/// +/// A utility class to generate a (compressed) archive at a target directory path. Accepts a +/// to filter out which files need to be archived. +/// +/// Upstream uses commons-compress. This port uses the .NET in-box archive writers: +/// for zip/jar, for tar (with +/// for tar.gz). XZ is not supported by the BCL and is left as TODO(port). +/// +public static class ArchiveUtil +{ + /// Internal utility to create an archive. + /// generic configured to write to the target archive file. + /// category of archive to generate based on target file extension. + /// copybara checkout path to the archive file. + /// glob to filter the set of files to be included in the archive. + /// + /// + public static void CreateArchive( + Stream os, ExtractType type, CheckoutPath archivePath, Glob? fileFilter) + { + switch (type) + { + case ExtractType.JAR: + case ExtractType.ZIP: + using (var zip = new ZipArchive(os, ZipArchiveMode.Create, leaveOpen: true)) + { + WriteFiles(archivePath, fileFilter, (relativePath, filePath) => + { + var entry = zip.CreateEntry(relativePath); + using var entryStream = entry.Open(); + using var source = File.OpenRead(filePath); + source.CopyTo(entryStream); + }); + } + break; + case ExtractType.TAR: + using (var tar = new TarWriter(os, leaveOpen: true)) + { + WriteFiles(archivePath, fileFilter, (relativePath, filePath) => + tar.WriteEntry(filePath, relativePath)); + } + break; + case ExtractType.TAR_GZ: + using (var gz = new GZipStream(os, CompressionMode.Compress, leaveOpen: true)) + using (var tar = new TarWriter(gz, leaveOpen: true)) + { + WriteFiles(archivePath, fileFilter, (relativePath, filePath) => + tar.WriteEntry(filePath, relativePath)); + } + break; + case ExtractType.TAR_XZ: + // TODO(port): XZ compression is not available in the .NET BCL. A third-party + // package (e.g. SharpCompress or XZ.NET) would be required to support TAR_XZ. + throw new ValidationException( + "TAR_XZ archives are not yet supported in the .NET port (no in-box XZ codec)."); + default: + throw new ValidationException( + $"Failed to get archive output stream for file type: {type}"); + } + } + + private static void WriteFiles( + CheckoutPath archivePath, Glob? fileFilter, Action writeEntry) + { + // Get the current working directory + string workdir = archivePath.GetCheckoutDir(); + + // Exclude the "archive file" itself from getting added to the archived bundle. + fileFilter ??= Glob.CreateGlob(ImmutableArray.Create("**")); + fileFilter = Glob.Difference( + fileFilter, Glob.CreateGlob(ImmutableArray.Create(archivePath.GetPath()))); + + var matcher = fileFilter.RelativeTo(workdir); + + foreach (string filePath in Directory.EnumerateFiles( + workdir, "*", SearchOption.AllDirectories)) + { + string full = PathOps.Normalize(Path.GetFullPath(filePath)); + if (!matcher.Matches(full)) + { + continue; + } + string relativePath = PathOps.Relativize(workdir, full); + writeEntry(relativePath, full); + } + } +} diff --git a/src/Copybara.Core/Authoring/Author.cs b/src/Copybara.Core/Authoring/Author.cs new file mode 100644 index 000000000..252d5d37e --- /dev/null +++ b/src/Copybara.Core/Authoring/Author.cs @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Authoring; + +/// +/// Represents the contributor of a change in the destination repository. A contributor can be either +/// an individual or a team. +/// +/// Author is lenient in name or email validation. +/// +[StarlarkBuiltin("author", Doc = "Represents the author of a change")] +public sealed class Author : IStarlarkValue +{ + private readonly string _name; + private readonly string _email; + + public Author(string name, string email) + { + _name = Preconditions.CheckNotNull(name); + _email = Preconditions.CheckNotNull(email); + } + + /// Returns the name of the author. + [StarlarkMethod("name", Doc = "The name of the author", StructField = true)] + public string Name => _name; + + /// Returns the email address of the author. + [StarlarkMethod("email", Doc = "The email of the author", StructField = true)] + public string Email => _email; + + /// + /// Returns the string representation of an author, which is the standard format + /// Name <email> used by most version control systems. + /// + public override string ToString() => $"{_name} <{_email}>"; + + public override bool Equals(object? o) + { + if (o is Author that) + { + // Authors with the same non-empty email are the same author. + return string.IsNullOrEmpty(_email) && string.IsNullOrEmpty(that._email) + ? string.Equals(_name, that._name) + : string.Equals(_email, that._email); + } + return false; + } + + /// Parse author from a String in the format of: "name <foo@bar.com>". + public static Author Parse(string authorStr) + { + try + { + return AuthorParser.Parse(authorStr); + } + catch (InvalidAuthorException e) + { + throw Starlark.Eval.Starlark.Errorf( + "Author '{0}' doesn't match the expected format 'name : {1}", + authorStr, + e.Message); + } + } + + public override int GetHashCode() => + string.IsNullOrEmpty(_email) ? _name.GetHashCode() : _email.GetHashCode(); +} diff --git a/src/Copybara.Core/Authoring/AuthorParser.cs b/src/Copybara.Core/Authoring/AuthorParser.cs new file mode 100644 index 000000000..49fb36e88 --- /dev/null +++ b/src/Copybara.Core/Authoring/AuthorParser.cs @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Concurrent; +using System.Text.RegularExpressions; +using Copybara.Common; + +namespace Copybara.Authoring; + +/// +/// A parser for the standard author format "Name <email>". +/// +/// This is the format used by most VCS (Git, Mercurial) and also by the Copybara configuration +/// itself. The parser is lenient: email can be empty, and it doesn't validate that is an +/// actual email. +/// +public class AuthorParser +{ + // Anchored to emulate Java Matcher.matches(), which requires the entire input to match. + private static readonly Regex AuthorPattern = + new(@"\A(?[^<]+)<(?[^>]*)>\z"); + + // The cache mirrors Guava's LoadingCache used by Copybara for repeated author loads. + private static readonly ConcurrentDictionary Cache = new(); + + /// Parses a Git author string into an . + public static Author Parse(string author) + { + Preconditions.CheckNotNull(author); + // Use a cache since repetitive load (thru --read-config-from-change) configs that + // define authors have a penalty because of the regex check/group. + if (Cache.TryGetValue(author, out var cached)) + { + return cached; + } + var parsed = InternalParse(author); + Cache[author] = parsed; + return parsed; + } + + private static Author InternalParse(string author) + { + if (IsInQuotes(author)) + { + author = author.Substring(1, author.Length - 2); // strip quotes + } + Match matcher = AuthorPattern.Match(author); + if (matcher.Success) + { + return new Author(matcher.Groups[1].Value.Trim(), matcher.Groups[2].Value.Trim()); + } + throw new InvalidAuthorException( + $"Invalid author '{author}'. Must be in the form of 'Name '"); + } + + private static bool IsInQuotes(string author) + { + // Equivalent to re2j pattern ("(\".+\")|(\'.+\')") with full-string match semantics: + // either a double-quoted or single-quoted string whose contents are at least one char. + if (author.Length >= 3 && author[0] == '"' && author[^1] == '"') + { + return true; + } + if (author.Length >= 3 && author[0] == '\'' && author[^1] == '\'') + { + return true; + } + return false; + } +} diff --git a/src/Copybara.Core/Authoring/Authoring.cs b/src/Copybara.Core/Authoring/Authoring.cs new file mode 100644 index 000000000..4f2ea4dec --- /dev/null +++ b/src/Copybara.Core/Authoring/Authoring.cs @@ -0,0 +1,296 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Authoring; + +/// +/// Represents the authors mapping between an origin and a destination. +/// +/// For a given author in the origin, always provides an author in the destination. +/// +[StarlarkBuiltin( + "authoring_class", + Doc = "The authors mapping between an origin and a destination")] +public sealed class Authoring : IStarlarkValue +{ + private readonly Author _defaultAuthor; + private readonly AuthoringMappingMode _mode; + private readonly IEvalThrowingPredicate _allowPredicate; + + public Authoring( + Author defaultAuthor, + AuthoringMappingMode mode, + IEvalThrowingPredicate allowPredicate) + { + _defaultAuthor = Preconditions.CheckNotNull(defaultAuthor); + _mode = mode; + _allowPredicate = Preconditions.CheckNotNull(allowPredicate); + } + + public Authoring(Author defaultAuthor, AuthoringMappingMode mode, ImmutableHashSet list) + : this(defaultAuthor, mode, new AllowlistPredicate(Preconditions.CheckNotNull(list))) + { + } + + /// Returns the mapping mode. + public AuthoringMappingMode GetMode() => _mode; + + /// + /// Returns the default author, used for squash workflows, + /// mode and for non-allowed authors. + /// + public Author GetDefaultAuthor() => _defaultAuthor; + + /// + /// Returns a predicate over allowed author identifiers. + /// + /// An identifier is typically an email but might have different representations depending + /// on the origin. + /// + public IEvalThrowingPredicate GetAllowPredicate() => _allowPredicate; + + /// Returns true if the user can be safely used. + public bool UseAuthor(string userId) => + _mode switch + { + AuthoringMappingMode.PassThru => true, + AuthoringMappingMode.Overwrite => false, + AuthoringMappingMode.Allowed => _allowPredicate.Test(userId), + _ => throw new InvalidOperationException($"Unexpected mode: {_mode}"), + }; + + /// Starlark Module for authoring. + [StarlarkBuiltin( + "authoring", + Doc = "The authors mapping between an origin and a destination")] + public sealed class Module : IStarlarkValue + { + [StarlarkMethod( + "overwrite", + Doc = + "Use the default author for all the submits in the destination. Note that some" + + " destinations might choose to ignore this author and use the current user" + + " running the tool (In other words they don't allow impersonation).")] + public Authoring Overwrite( + [Param(Name = "default", Named = true, Doc = "The default author for commits in the destination")] + string defaultAuthor) => + new(Author.Parse(defaultAuthor), AuthoringMappingMode.Overwrite, new RejectAllPredicate()); + + [StarlarkMethod( + "pass_thru", + Doc = "Use the origin author as the author in the destination, no filtering.")] + public Authoring PassThru( + [Param( + Name = "default", + Named = true, + Doc = + "The default author for commits in the destination. This is used" + + " in squash mode workflows or if author cannot be determined.")] + string defaultAuthor) => + new(Author.Parse(defaultAuthor), AuthoringMappingMode.PassThru, new RejectAllPredicate()); + + [StarlarkMethod( + "allowed", + Doc = "Create a list for an individual or team contributing code.", + UseStarlarkThread = true)] + public Authoring Allowed( + [Param( + Name = "default", + Named = true, + Doc = + "The default author for commits in the destination. This is used" + + " in squash mode workflows or when users are not on the list.")] + string defaultAuthor, + [Param( + Name = "allowlist", + Named = true, + DefaultValue = "None", + Doc = + "List of authors in the origin that are allowed to contribute code. The " + + "authors must be unique")] + IEnumerable? allowlist, + [Param( + Name = "allow_predicate", + Named = true, + DefaultValue = "None", + Doc = + "Starlark function to use to check if an author is allowed to contribute code." + + " The function should take a single argument (the author) and return" + + " true if the author is allowed, false otherwise. Allowlist is ignored if" + + " this is set.")] + IEvalThrowingPredicate? allowPred) + { + IEvalThrowingPredicate allowPredicate; + if (allowPred is null && allowlist is null) + { + throw Starlark.Eval.Starlark.Errorf( + "'allowed' function requires either an 'allowlist' or an 'allow_predicate' parameter."); + } + if (allowPred is not null) + { + allowPredicate = allowPred; + } + else + { + ImmutableHashSet allowedAuthors = CreateAllowlist(allowlist!); + allowPredicate = new AllowlistPredicate(allowedAuthors); + } + return new Authoring( + Author.Parse(defaultAuthor), AuthoringMappingMode.Allowed, allowPredicate); + } + + private static ImmutableHashSet CreateAllowlist(IEnumerable list) + { + var items = list.ToList(); + if (items.Count == 0) + { + throw Starlark.Eval.Starlark.Errorf( + "'allowed' function requires a non-empty 'allowlist' field. For default mapping," + + " use 'overwrite(...)' mode instead."); + } + var uniqueAuthors = new HashSet(); + foreach (var author in items) + { + if (!uniqueAuthors.Add(author)) + { + throw Starlark.Eval.Starlark.Errorf("Duplicated allowlist entry '{0}'", author); + } + } + return items.ToImmutableHashSet(); + } + } + + /// + /// Mode used for author mapping from origin to destination. + /// + /// This enum is our internal representation for the different Skylark built-in + /// functions. + /// + public enum AuthoringMappingMode + { + /// Corresponds with built-in function. + Overwrite, + + /// Corresponds with built-in function. + PassThru, + + /// Corresponds to built-in function. + Allowed, + } + + public override bool Equals(object? o) + { + if (ReferenceEquals(this, o)) + { + return true; + } + if (o is null || GetType() != o.GetType()) + { + return false; + } + var authoring = (Authoring)o; + return Equals(_defaultAuthor, authoring._defaultAuthor) + && _mode == authoring._mode + && Equals(_allowPredicate, authoring._allowPredicate); + } + + public override int GetHashCode() => HashCode.Combine(_defaultAuthor, _mode, _allowPredicate); + + public override string ToString() => + $"Authoring{{defaultAuthor={_defaultAuthor}, mode={_mode}, allowPredicate={_allowPredicate}}}"; + + /// A predicate that can throw . + public interface IEvalThrowingPredicate + { + bool Test(T input); + } + + /// + /// A predicate that uses a Starlark function to check if an author is allowed to be attributed + /// for code. + /// + /// + /// Java wraps a StarlarkCallable invoked on a StarlarkThread. Until those types + /// are ported, this holds a delegate that evaluates the author, preserving the type surface. + /// + public sealed class AllowPredicate : IEvalThrowingPredicate + { + private readonly Func _allowPred; + + public AllowPredicate(Func allowPred) + { + _allowPred = allowPred; + } + + public bool Test(string author) => _allowPred(author); + + public override bool Equals(object? obj) => + obj is AllowPredicate other && Equals(_allowPred, other._allowPred); + + public override int GetHashCode() => _allowPred.GetHashCode(); + } + + /// + /// A predicate that uses a list of allowed authors to check if an author is allowed to be + /// attributed for code. + /// + public sealed class AllowlistPredicate : IEvalThrowingPredicate + { + private readonly ImmutableHashSet _allowedAuthors; + + public AllowlistPredicate(ImmutableHashSet allowedAuthors) + { + _allowedAuthors = allowedAuthors; + } + + public ImmutableHashSet AllowedAuthors => _allowedAuthors; + + public bool Test(string author) => _allowedAuthors.Contains(author); + + public override bool Equals(object? obj) => + obj is AllowlistPredicate other && _allowedAuthors.SetEquals(other._allowedAuthors); + + public override int GetHashCode() + { + int hash = 0; + foreach (var a in _allowedAuthors) + { + hash ^= a.GetHashCode(); + } + return hash; + } + + public override string ToString() => + $"AllowlistPredicate{{allowedAuthors=[{string.Join(", ", _allowedAuthors)}]}}"; + } + + /// A predicate that always returns false for attribution. + public sealed class RejectAllPredicate : IEvalThrowingPredicate + { + public bool Test(string author) => false; + + public override bool Equals(object? obj) => obj is RejectAllPredicate; + + public override int GetHashCode() => typeof(RejectAllPredicate).GetHashCode(); + + public override string ToString() => "RejectAllPredicate{}"; + } +} diff --git a/src/Copybara.Core/Authoring/InvalidAuthorException.cs b/src/Copybara.Core/Authoring/InvalidAuthorException.cs new file mode 100644 index 000000000..cb950e37a --- /dev/null +++ b/src/Copybara.Core/Authoring/InvalidAuthorException.cs @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Authoring; + +/// +/// Indicates that the author does not conform to the expected format. +/// +/// This exception does not extend other exceptions ValidationException or +/// RepoException because a bad parsed author could come from a configuration or a repo, and +/// it's wrapped into the proper one by the caller. +/// +public class InvalidAuthorException : Exception +{ + internal InvalidAuthorException(string message) + : base(message) + { + } +} diff --git a/src/Copybara.Core/AutoPatchfileConfiguration.cs b/src/Copybara.Core/AutoPatchfileConfiguration.cs new file mode 100644 index 000000000..ee0846d19 --- /dev/null +++ b/src/Copybara.Core/AutoPatchfileConfiguration.cs @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara; + +/// Parameters for customizing auto patch file generation. +[StarlarkBuiltin( + "core.autopatch_config", + Doc = "The configuration that describes automatic patch file generation")] +public sealed class AutoPatchfileConfiguration : IStarlarkValue +{ + private readonly string? _header; + private readonly string _suffix; + private readonly string _directoryPrefix; + private readonly string? _directory; + private readonly bool _stripFilenames; + private readonly bool _stripLineNumbers; + private readonly Glob _glob; + + private AutoPatchfileConfiguration( + string? header, + string suffix, + string directoryPrefix, + string? directory, + bool stripFilenames, + bool stripLineNumbers, + Glob glob) + { + _header = header; + _suffix = suffix; + _directoryPrefix = directoryPrefix; + _directory = directory; + _stripFilenames = stripFilenames; + _stripLineNumbers = stripLineNumbers; + _glob = glob; + } + + public static AutoPatchfileConfiguration Create( + string? header, + string suffix, + string directoryPrefix, + string? directory, + bool stripFilenames, + bool stripLineNumbers, + Glob glob) => + new(header, suffix, directoryPrefix, directory, stripFilenames, stripLineNumbers, glob); + + public string? Header() => _header; + + public string Suffix() => _suffix; + + public string DirectoryPrefix() => _directoryPrefix; + + public string? Directory() => _directory; + + public bool StripFilenames() => _stripFilenames; + + public bool StripLineNumbers() => _stripLineNumbers; + + public Glob GlobValue() => _glob; +} diff --git a/src/Copybara.Core/BaselinesWithoutLabelVisitor.cs b/src/Copybara.Core/BaselinesWithoutLabelVisitor.cs new file mode 100644 index 000000000..f7a093000 --- /dev/null +++ b/src/Copybara.Core/BaselinesWithoutLabelVisitor.cs @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Revision; +using Copybara.Util; + +namespace Copybara; + +/// A visitor that finds all the parents that match the origin glob. +public class BaselinesWithoutLabelVisitor : IChangesVisitor +{ + private readonly List _result = new(); + private readonly int _limit; + private readonly Glob _originFiles; + private bool _skipFirst; + private readonly IRevision? _toSkip; + + public BaselinesWithoutLabelVisitor(Glob originFiles, int limit, IRevision? toSkip, bool skipFirst) + { + _originFiles = Preconditions.CheckNotNull(originFiles); + Preconditions.CheckArgument(limit > 0); + _limit = limit; + _toSkip = toSkip; + _skipFirst = skipFirst; + } + + public IReadOnlyList GetResult() => _result.ToImmutableArray(); + + public VisitResult Visit(Change change) + { + if (_skipFirst || (_toSkip != null && change.GetRevision().Equals(_toSkip))) + { + _skipFirst = false; + return VisitResult.Continue; + } + var files = change.GetChangeFiles(); + if (Glob.AffectsRoots(_originFiles.Roots(), files)) + { + _result.Add((T)(object)change.GetRevision()); + return _result.Count < _limit ? VisitResult.Continue : VisitResult.Terminate; + } + // This change only contains files that are not exported + return VisitResult.Continue; + } +} diff --git a/src/Copybara.Core/Buildozer/BuildozerBatch.cs b/src/Copybara.Core/Buildozer/BuildozerBatch.cs new file mode 100644 index 000000000..ba67328cf --- /dev/null +++ b/src/Copybara.Core/Buildozer/BuildozerBatch.cs @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Buildozer; +using Copybara.Common; +using BuildozerCommand = Copybara.Buildozer.BuildozerOptions.BuildozerCommand; + +namespace Copybara.Buildozer; + +/// A transformation that runs many buildozer transformation in batch. +public sealed class BuildozerBatch : IBuildozerTransformation +{ + private readonly BuildozerOptions _options; + private readonly WorkflowOptions _workflowOptions; + private readonly ImmutableArray _transformations; + + private BuildozerBatch( + BuildozerOptions options, + WorkflowOptions workflowOptions, + IEnumerable transformations) + { + _options = Preconditions.CheckNotNull(options); + _workflowOptions = Preconditions.CheckNotNull(workflowOptions); + _transformations = transformations.ToImmutableArray(); + } + + public TransformationStatus Transform(TransformWork work) + { + var commands = new List(); + foreach (IBuildozerTransformation transformation in _transformations) + { + transformation.BeforeRun(work); + commands.AddRange(transformation.GetCommands()); + } + try + { + _options.Run(work.GetConsole(), work.GetCheckoutDir(), commands); + return TransformationStatus.Success(); + } + catch (TargetNotFoundException e) + { + return TransformationStatus.Noop(e.Message); + } + } + + public ITransformation Reverse() => + throw new InvalidOperationException( + "Reverse should never be called for join transformations"); + + public string Describe() => + "buildozer batch of " + _transformations.Length + " buildozer transformations"; + + public bool CanJoin(ITransformation transformation) => IsBuildozer(transformation); + + internal static bool IsBuildozer(ITransformation transformation) => + transformation is IBuildozerTransformation; + + public ITransformation Join(ITransformation next) => + Join(_options, _workflowOptions, this, (IBuildozerTransformation)next); + + internal static BuildozerBatch Join( + BuildozerOptions buildozerOptions, + WorkflowOptions workflowOptions, + IBuildozerTransformation current, + IBuildozerTransformation next) + { + var transformationBuilder = ImmutableArray.CreateBuilder(); + if (current is BuildozerBatch currentBatch) + { + transformationBuilder.AddRange(currentBatch._transformations); + } + else + { + transformationBuilder.Add(current); + } + if (next is BuildozerBatch nextBatch) + { + transformationBuilder.AddRange(nextBatch._transformations); + } + else + { + transformationBuilder.Add(next); + } + return new BuildozerBatch( + buildozerOptions, workflowOptions, transformationBuilder.ToImmutable()); + } + + internal static BuildozerBatch JoinAll( + BuildozerOptions buildozerOptions, + WorkflowOptions workflowOptions, + IEnumerable transformations) + { + var transformationBuilder = ImmutableArray.CreateBuilder(); + foreach (IBuildozerTransformation transformation in transformations) + { + if (transformation is BuildozerBatch batch) + { + transformationBuilder.AddRange(batch._transformations); + } + else + { + transformationBuilder.Add(transformation); + } + } + return new BuildozerBatch( + buildozerOptions, workflowOptions, transformationBuilder.ToImmutable()); + } + + public void BeforeRun(TransformWork work) + { + foreach (IBuildozerTransformation transformation in _transformations) + { + transformation.BeforeRun(work); + } + } + + public IEnumerable GetCommands() + { + var result = new List(); + foreach (IBuildozerTransformation transformation in _transformations) + { + result.AddRange(transformation.GetCommands()); + } + return result; + } +} diff --git a/src/Copybara.Core/Buildozer/BuildozerCreate.cs b/src/Copybara.Core/Buildozer/BuildozerCreate.cs new file mode 100644 index 000000000..0b62d8100 --- /dev/null +++ b/src/Copybara.Core/Buildozer/BuildozerCreate.cs @@ -0,0 +1,166 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Buildozer; +using Copybara.Common; +using Copybara.Exceptions; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; +using BuildozerCommand = Copybara.Buildozer.BuildozerOptions.BuildozerCommand; + +namespace Copybara.Buildozer; + +/// +/// A transformation which creates a new build target and reverses to delete the same target. +/// +public sealed class BuildozerCreate : IBuildozerTransformation +{ + private readonly BuildozerOptions _options; + private readonly WorkflowOptions _workflowOptions; + private readonly Target _target; + private readonly string _ruleType; + private readonly RelativeTo _relativeTo; + private readonly ImmutableArray _commands; + + internal sealed class RelativeTo + { + internal readonly string Args; + + private static void ValidateTargetName(string targetName) + { + if (targetName.Contains(':')) + { + throw StarlarkRt.Errorf( + "unexpected : in target name (did you include the package by mistake?) - '{0}'", + targetName); + } + } + + internal RelativeTo(string before, string after) + { + if (before.Length != 0 && after.Length != 0) + { + throw new EvalException( + "cannot specify both 'before' and 'after' in the target create arguments"); + } + + if (before.Length != 0) + { + ValidateTargetName(before); + Args = "before " + before; + } + else if (after.Length != 0) + { + ValidateTargetName(after); + Args = "after " + after; + } + else + { + Args = ""; + } + } + + public override string ToString() => Args; + } + + internal BuildozerCreate( + BuildozerOptions options, + WorkflowOptions workflowOptions, + Target target, + string ruleType, + RelativeTo relativeTo, + IEnumerable commands) + { + _options = Preconditions.CheckNotNull(options); + _workflowOptions = Preconditions.CheckNotNull(workflowOptions); + _target = Preconditions.CheckNotNull(target); + _ruleType = Preconditions.CheckNotNull(ruleType); + _relativeTo = Preconditions.CheckNotNull(relativeTo); + _commands = commands.ToImmutableArray(); + } + + public TransformationStatus Transform(TransformWork work) + { + BeforeRun(work); + try + { + _options.Run(work.GetConsole(), work.GetCheckoutDir(), GetCommands()); + return TransformationStatus.Success(); + } + catch (TargetNotFoundException e) + { + // This should not happen for creation. If it happens, it is due to a file error. + throw new ValidationException(e.Message); + } + } + + public void BeforeRun(TransformWork work) + { + string buildFilePath = Path.Combine(work.GetCheckoutDir(), GetTargetBuildFile()); + if (!File.Exists(buildFilePath)) + { + // Alert the user that the package to contain this target doesn't have a BUILD file, since + // this may be a configuration error. + work.GetConsole().Info( + $"BUILD file to contain {_target} doesn't exist. Creating now."); + string? parent = Path.GetDirectoryName(buildFilePath); + if (parent != null) + { + Directory.CreateDirectory(parent); + } + File.WriteAllBytes(buildFilePath, Array.Empty()); + } + } + + private string GetTargetBuildFile() + { + string pkg = _target.GetPackage(); + // pkg can be empty (e.g. ":foo"), which should create targets in the workdir root, i.e. + // ./BUILD + return pkg + (pkg.Length == 0 ? "." : "") + "/BUILD"; + } + + public IEnumerable GetCommands() + { + var result = new List + { + new( + ImmutableArray.Create(GetTargetBuildFile()), + $"new {_ruleType} {_target.GetName()} {_relativeTo.Args}"), + }; + foreach (string command in _commands) + { + result.Add(new BuildozerCommand(ImmutableArray.Create(_target.ToString()), command)); + } + return result; + } + + public bool CanJoin(ITransformation transformation) => + BuildozerBatch.IsBuildozer(transformation); + + public ITransformation Join(ITransformation next) => + BuildozerBatch.Join(_options, _workflowOptions, this, (IBuildozerTransformation)next); + + public string Describe() => "buildozer.create " + _target; + + public ITransformation Reverse() => + new BuildozerDelete(_options, _workflowOptions, _target, this); + + public override string ToString() => + $"BuildozerCreate{{target={_target}, ruleType={_ruleType}, relativeTo={_relativeTo}," + + $" commands=[{string.Join(", ", _commands)}]}}"; +} diff --git a/src/Copybara.Core/Buildozer/BuildozerDelete.cs b/src/Copybara.Core/Buildozer/BuildozerDelete.cs new file mode 100644 index 000000000..f5c797ef4 --- /dev/null +++ b/src/Copybara.Core/Buildozer/BuildozerDelete.cs @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Buildozer; +using Copybara.Common; +using Copybara.Exceptions; +using BuildozerCommand = Copybara.Buildozer.BuildozerOptions.BuildozerCommand; + +namespace Copybara.Buildozer; + +/// +/// A transformation which deletes build target and reverses to create the same target. +/// +public sealed class BuildozerDelete : IBuildozerTransformation +{ + private readonly BuildozerOptions _options; + private readonly WorkflowOptions _workflowOptions; + private readonly Target _target; + private readonly BuildozerCreate? _recreateAs; + + internal BuildozerDelete( + BuildozerOptions options, + WorkflowOptions workflowOptions, + Target target, + BuildozerCreate? recreateAs) + { + _options = Preconditions.CheckNotNull(options); + _workflowOptions = Preconditions.CheckNotNull(workflowOptions); + _target = Preconditions.CheckNotNull(target); + _recreateAs = recreateAs; + } + + public TransformationStatus Transform(TransformWork work) + { + try + { + _options.Run(work.GetConsole(), work.GetCheckoutDir(), GetCommands()); + return TransformationStatus.Success(); + } + catch (TargetNotFoundException e) + { + return TransformationStatus.Noop(e.Message); + } + } + + public string Describe() => "buildozer.delete " + _target; + + public bool CanJoin(ITransformation transformation) => + BuildozerBatch.IsBuildozer(transformation); + + public ITransformation Join(ITransformation next) => + BuildozerBatch.Join(_options, _workflowOptions, this, (IBuildozerTransformation)next); + + public IEnumerable GetCommands() => + ImmutableArray.Create(new BuildozerCommand(_target.ToString(), "delete")); + + public ITransformation Reverse() + { + if (_recreateAs == null) + { + throw new NonReversibleValidationException( + "This buildozer.delete is not reversible. Please specify at least rule_type to make" + + " it reversible."); + } + return _recreateAs; + } + + public override string ToString() => + $"BuildozerDelete{{target={_target}, recreateAs={_recreateAs}}}"; +} diff --git a/src/Copybara.Core/Buildozer/BuildozerModify.cs b/src/Copybara.Core/Buildozer/BuildozerModify.cs new file mode 100644 index 000000000..96ed9b7b9 --- /dev/null +++ b/src/Copybara.Core/Buildozer/BuildozerModify.cs @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Buildozer; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Util.Console; +using BuildozerCommand = Copybara.Buildozer.BuildozerOptions.BuildozerCommand; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Buildozer; + +/// A transformation which runs one or more commands against a single target. +public sealed class BuildozerModify : IBuildozerTransformation +{ + private readonly BuildozerOptions _options; + private readonly WorkflowOptions _workflowOptions; + private readonly IReadOnlyList _targets; + private readonly ImmutableArray _commands; + + internal BuildozerModify( + BuildozerOptions options, + WorkflowOptions workflowOptions, + IReadOnlyList targets, + IEnumerable commands) + { + _options = Preconditions.CheckNotNull(options); + _workflowOptions = Preconditions.CheckNotNull(workflowOptions); + _targets = Preconditions.CheckNotNull(targets); + _commands = commands.ToImmutableArray(); + } + + public TransformationStatus Transform(TransformWork work) + { + Console console = work.GetConsole(); + try + { + _options.Run(console, work.GetCheckoutDir(), GetCommands()); + return TransformationStatus.Success(); + } + catch (TargetNotFoundException e) + { + return TransformationStatus.Noop(e.Message); + } + } + + public ITransformation Reverse() + { + var reverseCommands = new List(); + for (int i = _commands.Length - 1; i >= 0; i--) + { + reverseCommands.Add(_commands[i].Reverse()); + } + return new BuildozerModify(_options, _workflowOptions, _targets, reverseCommands); + } + + public IEnumerable GetCommands() + { + var result = new List(); + foreach (Command command in _commands) + { + result.Add(new BuildozerCommand(Target.AsStringList(_targets), command.ToString())); + } + return result; + } + + public bool CanJoin(ITransformation transformation) => + BuildozerBatch.IsBuildozer(transformation); + + public ITransformation Join(ITransformation next) => + BuildozerBatch.Join(_options, _workflowOptions, this, (IBuildozerTransformation)next); + + public string Describe() => "buildozer.modify [" + string.Join(", ", _targets) + "]"; + + public override string ToString() => + $"BuildozerModify{{target=[{string.Join(", ", _targets)}]," + + $" commands=[{string.Join(", ", _commands)}]}}"; +} diff --git a/src/Copybara.Core/Buildozer/BuildozerModule.cs b/src/Copybara.Core/Buildozer/BuildozerModule.cs new file mode 100644 index 000000000..00d82cb9c --- /dev/null +++ b/src/Copybara.Core/Buildozer/BuildozerModule.cs @@ -0,0 +1,356 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections; +using System.Collections.Immutable; +using Copybara.Buildozer; +using Copybara.Common; +using Copybara.Config; +using Copybara.Exceptions; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; +using Sequence = Starlark.Eval.Sequence; + +namespace Copybara.Buildozer; + +/// Skylark module for Buildozer-related functionality. +[StarlarkBuiltin( + "buildozer", + Doc = + "Module for Buildozer-related functionality such as creating and modifying BUILD targets.")] +public sealed class BuildozerModule : IStarlarkValue +{ + private readonly BuildozerOptions _buildozerOptions; + private readonly WorkflowOptions _workflowOptions; + private readonly BuildozerPrintExecutor _buildozerPrintExecutor; + + public BuildozerModule( + WorkflowOptions workflowOptions, + BuildozerOptions buildozerOptions, + GeneralOptions generalOptions) + { + _workflowOptions = Preconditions.CheckNotNull(workflowOptions); + _buildozerOptions = Preconditions.CheckNotNull(buildozerOptions); + _buildozerPrintExecutor = + BuildozerPrintExecutor.Create( + buildozerOptions, Preconditions.CheckNotNull(generalOptions).GetConsole()); + } + + private static ImmutableArray GetTargetList(object arg) + { + if (arg is string s) + { + return ImmutableArray.Create(Target.FromConfig(s)); + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (string target in SkylarkUtil.ConvertStringList(arg, "target")) + { + builder.Add(Target.FromConfig(target)); + } + return builder.ToImmutable(); + } + + private static ImmutableArray CoerceCommandList(IEnumerable commands) + { + var wrappedCommands = ImmutableArray.CreateBuilder(); + foreach (object? command in commands) + { + if (command is string s) + { + wrappedCommands.Add(Command.FromConfig(s, null)); + } + else if (command is Command c) + { + wrappedCommands.Add(c); + } + else + { + throw StarlarkRt.Errorf( + "Expected a string or buildozer.cmd, but got: {0}", command); + } + } + return wrappedCommands.ToImmutable(); + } + + [StarlarkMethod( + "create", + Doc = + "A transformation which creates a new build target and populates its " + + "attributes. This transform can reverse automatically to delete the target.")] + public BuildozerCreate Create( + [Param( + Name = "target", + Doc = + "Target to create, including the package, e.g. 'foo:bar'. The package can be " + + "'.' for the root BUILD file.", + Named = true)] + string target, + [Param( + Name = "rule_type", + Doc = "Type of this rule, for instance, java_library.", + Named = true)] + string ruleType, + [Param( + Name = "commands", + Doc = + "Commands to populate attributes of the target after creating it. Elements can" + + " be strings such as 'add deps :foo' or objects returned by buildozer.cmd.", + DefaultValue = "[]", + AllowedTypes = new[] { typeof(Sequence) }, + Named = true)] + object commands, + [Param( + Name = "before", + Doc = + "When supplied, causes this target to be created *before* the target named by" + + " 'before'", + Positional = false, + DefaultValue = "''", + Named = true)] + string before, + [Param( + Name = "after", + Doc = + "When supplied, causes this target to be created *after* the target named by" + + " 'after'", + Positional = false, + DefaultValue = "''", + Named = true)] + string after) + { + var commandStrings = new List(); + foreach (Command command in CoerceCommandList((IEnumerable)commands)) + { + commandStrings.Add(command.ToString()); + } + return new BuildozerCreate( + _buildozerOptions, + _workflowOptions, + Target.FromConfig(target), + ruleType, + new BuildozerCreate.RelativeTo(before, after), + commandStrings); + } + + private static void MustOmitRecreateParam(object expected, object actual, string paramName) + { + if (!expected.Equals(actual)) + { + throw StarlarkRt.Errorf( + "Parameter '{0}' is only used for reversible buildozer.delete transforms, but this" + + " buildozer.delete is not reversible. Specify 'rule_type' argument to make it" + + " reversible.", + paramName); + } + } + + [StarlarkMethod( + "delete", + Doc = + "A transformation which is the opposite of creating a build target. When run normally," + + " it deletes a build target. When reversed, it creates and prepares one.")] + public BuildozerDelete Delete( + [Param( + Name = "target", + Doc = "Target to delete, including the package, e.g. 'foo:bar'", + Named = true)] + string targetString, + [Param( + Name = "rule_type", + Doc = + "Type of this rule, for instance, java_library. Supplying this will cause this" + + " transformation to be reversible.", + DefaultValue = "''", + Named = true)] + string ruleType, + [Param( + Name = "recreate_commands", + Doc = + "Commands to populate attributes of the target after creating it. Elements can" + + " be strings such as 'add deps :foo' or objects returned by buildozer.cmd.", + Positional = false, + DefaultValue = "[]", + AllowedTypes = new[] { typeof(Sequence) }, + Named = true)] + object recreateCommands, + [Param( + Name = "before", + Doc = + "When supplied with rule_type and the transformation is reversed, causes this" + + " target to be created *before* the target named by 'before'", + Positional = false, + DefaultValue = "''", + Named = true)] + string before, + [Param( + Name = "after", + Doc = + "When supplied with rule_type and the transformation is reversed, causes this" + + " target to be created *after* the target named by 'after'", + Positional = false, + DefaultValue = "''", + Named = true)] + string after) + { + var commandStrings = new List(); + foreach (Command command in CoerceCommandList((IEnumerable)recreateCommands)) + { + commandStrings.Add(command.ToString()); + } + BuildozerCreate? recreateAs; + Target target = Target.FromConfig(targetString); + if (ruleType.Length == 0) + { + recreateAs = null; + MustOmitRecreateParam( + (object)ImmutableArray.Empty, ToListForCompare(recreateCommands), + "recreate_commands"); + MustOmitRecreateParam("", before, "before"); + MustOmitRecreateParam("", after, "after"); + } + else + { + recreateAs = + new BuildozerCreate( + _buildozerOptions, + _workflowOptions, + target, + ruleType, + new BuildozerCreate.RelativeTo(before, after), + commandStrings); + } + return new BuildozerDelete(_buildozerOptions, _workflowOptions, target, recreateAs); + } + + // Emulates Java's check that the (empty) default list argument was left untouched. + private static object ToListForCompare(object recreateCommands) + { + int count = 0; + foreach (object? unused in (IEnumerable)recreateCommands) + { + count++; + } + return count == 0 ? (object)ImmutableArray.Empty : recreateCommands; + } + + [StarlarkMethod( + "modify", + Doc = + "A transformation which runs one or more Buildozer commands against a single" + + " target expression. See http://go/buildozer for details on supported commands and" + + " target expression formats.")] + public BuildozerModify Modify( + [Param( + Name = "target", + AllowedTypes = new[] { typeof(string), typeof(Sequence) }, + Doc = "Specifies the target(s) against which to apply the commands. Can be a list.", + Named = true)] + object target, + [Param( + Name = "commands", + Doc = + "Commands to apply to the target(s) specified. Elements can" + + " be strings such as 'add deps :foo' or objects returned by buildozer.cmd.", + AllowedTypes = new[] { typeof(Sequence) }, + Named = true)] + object commands) + { + var commandList = CoerceCommandList((IEnumerable)commands); + if (commandList.Length == 0) + { + throw StarlarkRt.Errorf("at least one element required in 'commands' argument"); + } + return new BuildozerModify( + _buildozerOptions, _workflowOptions, GetTargetList(target), commandList); + } + + [StarlarkMethod( + "cmd", + Doc = + "Creates a Buildozer command. You can specify the reversal with the 'reverse' " + + "argument.")] + public Command Cmd( + [Param( + Name = "forward", + Doc = "Specifies the Buildozer command, e.g. 'replace deps :foo :bar'", + Named = true)] + string forward, + [Param( + Name = "reverse", + AllowedTypes = new[] { typeof(string), typeof(NoneType) }, + Doc = + "The reverse of the command. This is only required if the given command cannot be" + + " reversed automatically and the reversal of this command is required by" + + " some workflow or Copybara check. The following commands are automatically" + + " reversible:
  • add
  • remove (when used to remove element" + + " from list i.e. 'remove srcs foo.cc'
  • replace
", + DefaultValue = "None", + Named = true)] + object? reverse) + { + return Command.FromConfig(forward, SkylarkUtil.ConvertOptionalString(reverse)); + } + + [StarlarkMethod( + "print", + Doc = + "Executes a buildozer print command and returns the output. This is designed to be used" + + " in the context of a transform")] + public string Print( + [Param( + Name = "ctx", + Doc = "The TransformWork object", + AllowedTypes = new[] { typeof(TransformWork) }, + Named = true)] + TransformWork ctx, + [Param(Name = "attr", Doc = "The attribute from the target rule to print.", Named = true)] + string attr, + [Param(Name = "target", Doc = "The target to print from.", Named = true)] + string target) + { + return _buildozerPrintExecutor.Run(ctx.GetCheckoutDir(), attr, target); + } + + [StarlarkMethod( + "batch", + Doc = "Combines a list of buildozer transforms into a single batch transformation.")] + public BuildozerBatch Batch( + [Param( + Name = "transforms", + Doc = "The list of buildozer transforms to combine.", + AllowedTypes = new[] { typeof(Sequence) }, + Named = true)] + object transforms) + { + var builder = ImmutableArray.CreateBuilder(); + foreach (object? transform in (IEnumerable)transforms) + { + if (transform is IBuildozerTransformation t) + { + builder.Add(t); + } + else + { + throw StarlarkRt.Errorf( + "Expected a buildozer transform, but got: {0}", + transform?.GetType().Name ?? "null"); + } + } + return BuildozerBatch.JoinAll(_buildozerOptions, _workflowOptions, builder.ToImmutable()); + } +} diff --git a/src/Copybara.Core/Buildozer/BuildozerOptions.cs b/src/Copybara.Core/Buildozer/BuildozerOptions.cs new file mode 100644 index 000000000..c27e67e2e --- /dev/null +++ b/src/Copybara.Core/Buildozer/BuildozerOptions.cs @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text; +using System.Text.RegularExpressions; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Format; +using Copybara.Util; +using Copybara.Util.Console; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Command = Copybara.Util.Command; +using CommandException = Copybara.Util.CommandException; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Buildozer; + +/// Specifies how Buildozer is executed. +public sealed class BuildozerOptions : IOption +{ + private static readonly ILogger Logger = NullLogger.Instance; + + private static readonly Regex TargetNotFound = new( + @".*error while executing commands \[.+\] on target (?.* not found)", + RegexOptions.Compiled); + + private readonly GeneralOptions _generalOptions; + private readonly BuildifierOptions _buildifierOptions; + private readonly WorkflowOptions _workflowOptions; + + public BuildozerOptions( + GeneralOptions generalOptions, + BuildifierOptions buildifierOptions, + WorkflowOptions workflowOptions) + { + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _buildifierOptions = Preconditions.CheckNotNull(buildifierOptions); + _workflowOptions = workflowOptions; + } + + [Flag( + "--buildozer-bin", + "Binary to use for buildozer (Default is /usr/bin/buildozer)", + Hidden = true)] + public string BuildozerBin { get; set; } = "/usr/bin/buildozer"; + + private void LogError(Console console, CommandOutput output) + { + Consoles.ErrorLogLines(console, "buildozer stdout: ", output.GetStdout()); + Consoles.ErrorLogLines(console, "buildozer stderr: ", output.GetStderr()); + } + + public sealed class BuildozerCommand + { + private readonly IReadOnlyList _targets; + private readonly string _cmd; + + internal BuildozerCommand(IReadOnlyList targets, string cmd) + { + _targets = Preconditions.CheckNotNull(targets); + _cmd = Preconditions.CheckNotNull(cmd); + } + + internal BuildozerCommand(string targets, string cmd) + { + _targets = ImmutableArray.Create(Preconditions.CheckNotNull(targets)); + _cmd = Preconditions.CheckNotNull(cmd); + } + + public override string ToString() => _cmd + "|" + string.Join('|', _targets); + } + + /// + /// + internal void Run(Console console, string checkoutDir, IEnumerable commands) + { + string unused = RunCaptureOutput(console, checkoutDir, commands); + } + + /// Runs buildozer with the given commands. + /// + /// + internal string RunCaptureOutput( + Console console, string checkoutDir, IEnumerable commands) + { + var commandList = commands.ToList(); + var args = new List + { + BuildozerBin, + "-buildifier=" + _buildifierOptions.BuildifierBin, + }; + + // We only use -k in keep going mode because it shows less errors (http://b/69386431) + if (_workflowOptions.IgnoreNoop) + { + args.Add("-k"); + } + args.Add("-f"); + args.Add("-"); + try + { + var cmd = new Copybara.Util.Command(args.ToArray(), null, checkoutDir); + CommandOutputWithStatus output = _generalOptions.NewCommandRunner(cmd) + .WithVerbose(_generalOptions.IsVerbose()) + .WithInput(Encoding.UTF8.GetBytes(string.Join('\n', commandList))) + .Execute(); + if (output.GetStdout().Length != 0) + { + Logger.LogInformation("buildozer stdout: {Stdout}", output.GetStdout()); + } + if (output.GetStderr().Length != 0) + { + Logger.LogInformation("buildozer stderr: {Stderr}", output.GetStderr()); + } + return output.GetStdout(); + } + catch (BadExitStatusWithOutputException e) + { + // Don't print the output for common/known errors. + if (_generalOptions.IsVerbose()) + { + LogError(console, e.GetOutput()); + } + if (e.GetResult().TerminationStatus.GetExitCode() == 3) + { + // Buildozer exits with code == 3 when the build file was not modified and no output + // was generated. This happens with expressions that match multiple targets, like + // :%java_library + throw new TargetNotFoundException( + CommandsMessage("Buildozer could not find a target for", commandList)); + } + if (e.GetResult().TerminationStatus.GetExitCode() == 2) + { + var errors = e.GetOutput().GetStderr() + .Split('\n') + .Where(s => !(s.Length == 0 || s.StartsWith("fixed ", StringComparison.Ordinal))) + .ToList(); + var notFoundMsg = new List(); + bool allNotFound = true; + foreach (string error in errors) + { + Match matcher = TargetNotFound.Match(error); + if (matcher.Success) + { + notFoundMsg.Add( + $"Buildozer could not find a target for {matcher.Groups["error"].Value}"); + } + else if (error.Contains("no such file or directory") + || error.Contains("not a directory")) + { + notFoundMsg.Add("Buildozer could not find build file: " + error); + } + else + { + allNotFound = false; + } + } + if (allNotFound) + { + throw new TargetNotFoundException(string.Join('\n', notFoundMsg)); + } + } + // Otherwise we have already printed above. + if (!_generalOptions.IsVerbose()) + { + LogError(console, e.GetOutput()); + } + throw new ValidationException( + string.Format( + "{0}\nCommand stderr:{1}", + CommandsMessage("Failed to execute buildozer with args", commandList), + e.GetOutput().GetStderr()), + e); + } + catch (CommandException e) + { + string message = string.Format( + "Error '{0}' running buildozer command: {1}", + e.Message, + e.GetCommand()); + console.Error(message); + throw new ValidationException(message, e); + } + } + + private static string CommandsMessage(string prefix, IEnumerable commands) => + prefix + ":\n " + string.Join("\n ", commands); +} diff --git a/src/Copybara.Core/Buildozer/BuildozerPrintExecutor.cs b/src/Copybara.Core/Buildozer/BuildozerPrintExecutor.cs new file mode 100644 index 000000000..e2a8f9ebd --- /dev/null +++ b/src/Copybara.Core/Buildozer/BuildozerPrintExecutor.cs @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Buildozer; +using Copybara.Exceptions; +using BuildozerCommand = Copybara.Buildozer.BuildozerOptions.BuildozerCommand; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Buildozer; + +/// A class that can run a 'buildozer print' command. +public class BuildozerPrintExecutor +{ + private readonly BuildozerOptions _options; + private readonly Console _console; + + private BuildozerPrintExecutor(BuildozerOptions options, Console console) + { + _options = options; + _console = console; + } + + public static BuildozerPrintExecutor Create(BuildozerOptions options, Console console) => + new(options, console); + + /// + /// Runs a Buildozer print command. + /// + /// The checkout directory to run in. + /// The attribute from the target rule to print. + /// The target to print from. + /// A string with the buildozer print output. + /// If there is an issue running buildozer print. + public string Run(string checkoutDir, string attr, string target) + { + try + { + var command = new BuildozerCommand(target, $"print {attr}"); + return _options.RunCaptureOutput( + _console, checkoutDir, ImmutableArray.Create(command)); + } + catch (TargetNotFoundException e) + { + throw new ValidationException("Buildozer could not find the specified target", e); + } + } +} diff --git a/src/Copybara.Core/Buildozer/BuildozerTransformation.cs b/src/Copybara.Core/Buildozer/BuildozerTransformation.cs new file mode 100644 index 000000000..6f57d9055 --- /dev/null +++ b/src/Copybara.Core/Buildozer/BuildozerTransformation.cs @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Buildozer; + +namespace Copybara.Buildozer; + +/// +/// Common interface implemented by all buildozer transformations. +/// +/// Used by to batch multiple invocations in one buildozer cli +/// call. +/// +public interface IBuildozerTransformation : ITransformation +{ + /// Actions to run before calling buildozer. For example creating files. + void BeforeRun(TransformWork work) + { + } + + /// List of commands to execute. + IEnumerable GetCommands(); +} diff --git a/src/Copybara.Core/Buildozer/Command.cs b/src/Copybara.Core/Buildozer/Command.cs new file mode 100644 index 000000000..e9b2afc13 --- /dev/null +++ b/src/Copybara.Core/Buildozer/Command.cs @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Copybara.Exceptions; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara.Buildozer; + +/// Represents a possibly-reversible Buildozer command. +[StarlarkBuiltin("Command", Doc = "Buildozer command type")] +public sealed class Command : IStarlarkPrintableValue +{ + private readonly string _command; + private readonly string? _reverse; + + private Command(string command, string? reverse) + { + _command = Preconditions.CheckNotNull(command); + Preconditions.CheckArgument(command.Trim().Length != 0, "Found empty command"); + Preconditions.CheckArgument( + reverse == null || reverse.Trim().Length != 0, + "Found empty reversal command. Command was: {0}", + command); + + _reverse = reverse; + new ArgValidator(command).Validate(); + if (reverse != null) + { + new ArgValidator(reverse).Validate(); + } + } + + internal static Command FromConfig(string command, string? reverse) + { + if (reverse == null) + { + List components = command + .Split(' ', 2, StringSplitOptions.RemoveEmptyEntries) + .ToList(); + if (components.Count == 2) + { + reverse = ReverseArgs(components[0], components[1]); + } + } + + try + { + return new Command(command, reverse); + } + catch (ArgumentException ex) + { + throw new EvalException(ex.Message, ex); + } + } + + private sealed class ArgValidator + { + private readonly List _argv; + + internal ArgValidator(string command) + { + _argv = SplitArgv(command); + } + + private void ValidateCount(bool valid, string requirement) + { + Preconditions.CheckArgument( + valid, + "'{0}' requires {1}, but got: {2}", + _argv[0], + requirement, + ArgCount()); + } + + private int ArgCount() => _argv.Count - 1; + + internal void Validate() + { + Preconditions.CheckArgument( + _argv.Count != 0, "Expected an operation, but got empty string."); + switch (_argv[0]) + { + case "del_subinclude": + case "rename": + case "copy": + case "copy_no_overwrite": + ValidateCount(ArgCount() == 2, "exactly 2 arguments"); + break; + case "fix": + case "print": + case "remove_comment": + break; // can take 0+ + case "replace_subinclude": + case "move": + ValidateCount(ArgCount() >= 3, "at least 3 arguments"); + break; + case "delete": + ValidateCount(ArgCount() == 0, "exactly 0 arguments"); + break; + case "replace": + ValidateCount(ArgCount() == 3, "exactly 3 arguments"); + break; + case "comment": + case "remove": + case "set": + case "set_if_absent": + ValidateCount(ArgCount() >= 1, "at least 1 argument"); + break; + case "add": + case "new_load": + case "new": + ValidateCount(ArgCount() >= 2, "at least 2 arguments"); + break; + default: + // We assume that all unary operations are covered. + Preconditions.CheckArgument( + ArgCount() > 1, "Expected an operation, but got '{0}'.", _argv[0]); + break; + } + } + } + + public bool IsImmutable() => true; + + public void Repr(Printer printer, StarlarkSemantics semantics) + { + printer.Append( + string.Format("buildozer.cmd({0}, reverse = {1})", _command, _reverse)); + } + + /// + /// Returns the command and arguments concatenated, which can be passed directly to Buildozer. + /// + public override string ToString() => _command; + + /// Returns the reverse version of this command. + /// if this instance is not reversible + internal Command Reverse() + { + if (_reverse == null) + { + throw new NonReversibleValidationException( + "The current command is not auto-reversible and a reverse was not provided: " + + _command); + } + + return new Command(_reverse, _command); + } + + /// Calculates the reversal of a command whose reversal has not been manually specified. + private static string? ReverseArgs(string commandName, string args) + { + switch (commandName) + { + case "add": + return "remove " + args; + case "remove": + if (args.Contains(' ')) + { + // Do not reverse 'remove attr' operation. Only 'remove attr value' + return "add " + args; + } + return null; + case "replace": + List reverseArgs = SplitArgv(args); + if (reverseArgs.Count != 3) + { + throw StarlarkRt.Errorf( + "Cannot reverse '{0} {1}', expected three arguments, but found {2}.", + commandName, args, reverseArgs.Count); + } + (reverseArgs[1], reverseArgs[2]) = (reverseArgs[2], reverseArgs[1]); + return "replace " + string.Join(' ', reverseArgs); + } + return null; + } + + private static List SplitArgv(string argv) => + argv.Split(' ', StringSplitOptions.RemoveEmptyEntries).ToList(); +} diff --git a/src/Copybara.Core/Buildozer/Target.cs b/src/Copybara.Core/Buildozer/Target.cs new file mode 100644 index 000000000..e1a57209e --- /dev/null +++ b/src/Copybara.Core/Buildozer/Target.cs @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.RegularExpressions; +using Copybara.Common; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara.Buildozer; + +/// Specifies a target, including the package and name of target. +internal sealed class Target +{ + private static readonly Regex TargetNamePattern = new("^[^:]*:[^:]+$", RegexOptions.Compiled); + + private readonly string _pkg; + private readonly string _name; + + private Target(string[] components) + { + Preconditions.CheckArgument( + components.Length == 2, "{0}", string.Join(", ", components)); + + _pkg = Preconditions.CheckNotNull(components[0]); + _name = Preconditions.CheckNotNull(components[1]); + } + + public string GetPackage() => _pkg; + + public string GetName() => _name; + + public override string ToString() => _pkg + ":" + _name; + + /// + /// Parses a target specified in configuration. + /// + /// target specified in the form PKG:TARGET_NAME + /// if is not formatted correctly + internal static Target FromConfig(string configString) + { + if (configString.StartsWith("/", StringComparison.Ordinal)) + { + throw new EvalException("target must be relative and not start with '/' or '//'"); + } + if (!TargetNamePattern.IsMatch(configString)) + { + throw new EvalException( + "target must be in the form of {PKG}:{TARGET_NAME}, e.g. foo/bar:baz"); + } + return new Target(configString.Split(':', 2)); + } + + internal static ImmutableArray AsStringList(IReadOnlyList targets) => + targets.Select(t => t.ToString()).ToImmutableArray(); +} diff --git a/src/Copybara.Core/Buildozer/TargetNotFoundException.cs b/src/Copybara.Core/Buildozer/TargetNotFoundException.cs new file mode 100644 index 000000000..77da1bdd1 --- /dev/null +++ b/src/Copybara.Core/Buildozer/TargetNotFoundException.cs @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Buildozer; + +/// Error thrown when a target couldn't be found during a Buildozer transformation. +public class TargetNotFoundException : Exception +{ + internal TargetNotFoundException(string msg) + : base(msg) + { + } +} diff --git a/src/Copybara.Core/ChangeMessage.cs b/src/Copybara.Core/ChangeMessage.cs new file mode 100644 index 000000000..39d7c238b --- /dev/null +++ b/src/Copybara.Core/ChangeMessage.cs @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2016 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text; +using Copybara.Common; +using Copybara.Exceptions; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara; + +/// +/// An object that represents a well formed message: No superfluous new lines, a group of labels, +/// etc. +/// +/// This class is immutable. +/// +[StarlarkBuiltin( + "ChangeMessage", + Doc = "Represents a well formed parsed change message with its associated labels.")] +public sealed class ChangeMessage : IStarlarkValue +{ + private const string DoubleNewline = "\n\n"; + private const string DashDashSeparator = "\n--\n"; + + private readonly string _text; + private readonly string _groupSeparator; + private readonly ImmutableArray _labels; + + private ChangeMessage(string text, string groupSeparator, IReadOnlyList labels) + { + _text = text.Trim('\n'); + _groupSeparator = Preconditions.CheckNotNull(groupSeparator); + _labels = Preconditions.CheckNotNull(labels).ToImmutableArray(); + } + + /// + /// Create a new message object looking for labels in just the last paragraph. + /// + /// Use this for Copybara well-formed messages. + /// + public static ChangeMessage ParseMessage(string message) + { + string trimMsg = message.Trim('\n'); + int doubleNewLine = trimMsg.LastIndexOf(DoubleNewline, StringComparison.Ordinal); + int dashDash = trimMsg.LastIndexOf(DashDashSeparator, StringComparison.Ordinal); + if (doubleNewLine == -1 && dashDash == -1) + { + // Empty message like "\n\nfoo: bar" or "\n\nfoo bar baz" + if (message.StartsWith(DoubleNewline, StringComparison.Ordinal)) + { + return new ChangeMessage("", DoubleNewline, LinesAsLabels(trimMsg)); + } + return new ChangeMessage(trimMsg, DoubleNewline, new List()); + } + else if (doubleNewLine > dashDash) + { + return new ChangeMessage( + trimMsg.Substring(0, doubleNewLine), + DoubleNewline, + LinesAsLabels(trimMsg.Substring(doubleNewLine + 2))); + } + else + { + return new ChangeMessage( + trimMsg.Substring(0, dashDash), + DashDashSeparator, + LinesAsLabels(trimMsg.Substring(dashDash + 4))); + } + } + + /// + /// Create a new message object treating all the lines as possible labels instead of looking just + /// in the last paragraph for labels. + /// + public static ChangeMessage ParseAllAsLabels(string message) + { + Preconditions.CheckNotNull(message); + return new ChangeMessage("", DoubleNewline, LinesAsLabels(message)); + } + + private static List LinesAsLabels(string message) + { + Preconditions.CheckNotNull(message); + return message.TrimEnd('\n').Split('\n').Select(line => new LabelFinder(line)).ToList(); + } + + [StarlarkMethod("first_line", Doc = "First line of this message", StructField = true)] + public string FirstLine() + { + int idx = _text.IndexOf('\n'); + return idx == -1 ? _text : _text.Substring(0, idx); + } + + [StarlarkMethod( + "text", + Doc = "The text description this message, not including the labels.", + StructField = true)] + public string GetText() => _text; + + public IReadOnlyList GetLabels() => _labels; + + /// + /// Returns all the labels in the message. If a label appears multiple times, it respects the + /// order of appearance. + /// + public ImmutableListMultimap LabelsAsMultimap() + { + var result = ImmutableListMultimap.CreateBuilder(); + foreach (var label in _labels) + { + if (label.IsLabel()) + { + result.Put(label.GetName(), label.GetValue()); + } + } + return result.Build(); + } + + [StarlarkMethod( + "label_values", + Doc = "Returns a list of values associated with the label name.")] + public IReadOnlyList GetLabelValues( + [Param(Name = "label_name", Named = true, Doc = "The label name.")] string labelName) + { + var localLabels = LabelsAsMultimap(); + if (localLabels.ContainsKey(labelName)) + { + return localLabels.Get(labelName); + } + return ImmutableArray.Empty; + } + + public ChangeMessage WithLabel(string name, string separator, string value) + { + var newLabels = new List(_labels); + // Add an additional line if none of the previous elements are labels + if (newLabels.Count != 0 && !newLabels.Any(l => l.IsLabel())) + { + newLabels.Add(new LabelFinder("")); + } + newLabels.Add(new LabelFinder( + ValidateLabelName(name) + Preconditions.CheckNotNull(separator) + + Preconditions.CheckNotNull(value))); + return new ChangeMessage(_text, _groupSeparator, newLabels); + } + + public ChangeMessage WithReplacedLabel(string labelName, string separator, string value) + { + ValidateLabelName(labelName); + var newLabels = _labels + .Select(label => label.IsLabel(labelName) + ? new LabelFinder(labelName + separator + value) + : label) + .ToList(); + return new ChangeMessage(_text, _groupSeparator, newLabels); + } + + public ChangeMessage WithNewOrReplacedLabel(string labelName, string separator, string value) + { + ValidateLabelName(labelName); + var newLabels = new List(); + bool wasReplaced = false; + + foreach (var originalLabel in _labels) + { + if (originalLabel.IsLabel(labelName)) + { + newLabels.Add(new LabelFinder(labelName + separator + value)); + wasReplaced = true; + } + else + { + newLabels.Add(originalLabel); + } + } + + var newChangeMessage = new ChangeMessage(_text, _groupSeparator, newLabels); + if (!wasReplaced) + { + return newChangeMessage.WithLabel(labelName, separator, value); + } + return newChangeMessage; + } + + /// Filters out all labels that do not match . + public ChangeMessage WithLabelsFilteredBy(Func predicate) + { + var filteredLabels = _labels.Where(predicate).ToList(); + return new ChangeMessage(_text, _groupSeparator, filteredLabels); + } + + /// Remove a label by name if it exists. + public ChangeMessage WithRemovedLabelByName(string name) + { + ValidateLabelName(name); + var filteredLabels = _labels.Where(label => !label.IsLabel(name)).ToList(); + return new ChangeMessage(_text, _groupSeparator, filteredLabels); + } + + /// Remove a label by name and value if it exists. + public ChangeMessage WithRemovedLabelByNameAndValue(string name, string value) + { + ValidateLabelName(name); + var filteredLabels = _labels + .Where(label => !label.IsLabel(name) || !label.GetValue().Equals(value)) + .ToList(); + return new ChangeMessage(_text, _groupSeparator, filteredLabels); + } + + private static string ValidateLabelName(string label) + { + ValidationException.CheckCondition( + LabelFinder.VALID_LABEL.IsMatch(label), "Label '{0}' is not a valid label", label); + return label; + } + + /// Set the text part of the message, leaving the labels untouched. + public ChangeMessage WithText(string text) => + new(text.Trim('\n'), _groupSeparator, _labels); + + public override string ToString() + { + var sb = new StringBuilder(); + + if (_text.Length != 0) + { + sb.Append(_text).Append(_labels.Length == 0 ? "\n" : _groupSeparator); + } + foreach (var label in _labels) + { + sb.Append(label.GetLine()).Append('\n'); + } + // Let's normalize in case parseAllAsLabels was used and all the labels were removed. + return sb.ToString().Trim('\n') + '\n'; + } +} diff --git a/src/Copybara.Core/ChangeVisitable.cs b/src/Copybara.Core/ChangeVisitable.cs new file mode 100644 index 000000000..0a9f61240 --- /dev/null +++ b/src/Copybara.Core/ChangeVisitable.cs @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Revision; + +namespace Copybara; + +/// The result type for the function passed to visitChanges. +public enum VisitResult +{ + /// + /// Continue. If more changes are available for visiting, the origin will call again the function + /// with the next changes. + /// + Continue, + + /// + /// Stop. Origin will not pass more changes to the visitor function. Usually used because the + /// function found what it was looking for (for example a commit with a label). + /// + Terminate, +} + +/// +/// A visitor of changes. An implementation of this interface is provided to visitChanges +/// methods to visit changes in Origin or Destination history. +/// +public interface IChangesVisitor +{ + /// + /// Invoked for each change found. The implementation can choose to cancel the visitation by + /// returning . + /// + VisitResult Visit(Change input); +} + +/// A visitor of changes that only receives changes that match any of the passed labels. +public interface IChangesLabelVisitor +{ + /// + /// Invoked for each change found that matches the labels. + /// + /// Note that the can be disjoint with the labels in + /// , since labels might be stored with a different string format. + /// + VisitResult Visit(Change input, IReadOnlyDictionary matchedLabels); +} + +/// +/// An interface stating that the implementing class accepts child visitors to explore repository +/// state beyond the changes being migrated. +/// +/// the revision type. +public interface IChangeVisitable + where R : class, IRevision +{ + /// + /// Visit the parents of the revision and call the visitor for each + /// change. The visitor can stop the stream of changes at any moment by returning + /// . + /// + /// It is up to the Origin how and what changes it provides to the function. + /// + void VisitChanges(R? start, IChangesVisitor visitor); + + /// Visit only changes that contain any of the labels in . + void VisitChangesWithAnyLabel( + R? start, IReadOnlyCollection labels, IChangesLabelVisitor visitor) + { + VisitChanges(start, new AnyLabelAdapter(labels, visitor)); + } + + private sealed class AnyLabelAdapter : IChangesVisitor + { + private readonly IReadOnlyCollection _labels; + private readonly IChangesLabelVisitor _visitor; + + public AnyLabelAdapter(IReadOnlyCollection labels, IChangesLabelVisitor visitor) + { + _labels = labels; + _visitor = visitor; + } + + public VisitResult Visit(Change input) + { + // We could return all the label values, but this is really only used for RevId-like ones + // and last is good enough for now. + var labels = input.GetLabels(); + var copy = ImmutableDictionary.CreateBuilder(); + foreach (var key in labels.Keys) + { + if (_labels.Contains(key)) + { + var values = labels.Get(key); + copy[key] = values[values.Length - 1]; + } + } + if (copy.Count == 0) + { + return VisitResult.Continue; + } + return _visitor.Visit(input, copy.ToImmutable()); + } + } +} diff --git a/src/Copybara.Core/CheckoutFileSystem.cs b/src/Copybara.Core/CheckoutFileSystem.cs new file mode 100644 index 000000000..da87e25c2 --- /dev/null +++ b/src/Copybara.Core/CheckoutFileSystem.cs @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text; +using Copybara.Common; +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara; + +/// Common Starlark methods that allow users to manipulate paths of the workdir/checkoutPath. +public class CheckoutFileSystem : IStarlarkValue +{ + private readonly string _checkoutDir; + + public CheckoutFileSystem(string checkoutDir) + { + _checkoutDir = Preconditions.CheckNotNull(checkoutDir); + } + + [StarlarkMethod("new_path", Doc = "Create a new path")] + public CheckoutPath NewPath( + [Param( + Name = "path", + Doc = "The string representing the path, relative to the checkout root directory")] + string path) => + CheckoutPath.CreateWithCheckoutDir(path, _checkoutDir); + + [StarlarkMethod("create_symlink", Doc = "Create a symlink")] + public void CreateSymlink( + [Param(Name = "link", Doc = "The link path")] CheckoutPath link, + [Param(Name = "target", Doc = "The target path")] CheckoutPath target) + { + try + { + string linkFullPath = AsCheckoutPath(link); + // Verify target is inside checkout dir + _ = AsCheckoutPath(target); + + if (System.IO.File.Exists(linkFullPath) || System.IO.Directory.Exists(linkFullPath)) + { + string kind = System.IO.Directory.Exists(linkFullPath) + ? " and is a directory" + : (System.IO.File.GetAttributes(linkFullPath) & System.IO.FileAttributes.ReparsePoint) != 0 + ? " and is a symlink" + : System.IO.File.Exists(linkFullPath) + ? " and is a regular file" + : " and we don't know what kind of file is"; + throw StarlarkRt.Errorf("'{0}' already exist{1}", link.GetPath(), kind); + } + + string? linkParent = PathOps.GetParent(link.GetPath()); + string relativized = linkParent == null + ? target.GetPath() + : PathOps.Relativize(linkParent, target.GetPath()); + string? fullParent = PathOps.GetParent(linkFullPath); + if (fullParent != null) + { + System.IO.Directory.CreateDirectory(fullParent); + } + + System.IO.File.CreateSymbolicLink(linkFullPath, relativized); + } + catch (System.IO.IOException e) + { + throw StarlarkRt.Errorf("Cannot create symlink: {0}", e.Message); + } + } + + [StarlarkMethod("write_path", Doc = "Write an arbitrary string to a path (UTF-8 will be used)")] + public void WritePath( + [Param(Name = "path", Doc = "The Path to write to")] CheckoutPath path, + [Param(Name = "content", Doc = "The content of the file")] string content) + { + string fullPath = AsCheckoutPath(path); + string? parent = PathOps.GetParent(fullPath); + if (parent != null) + { + System.IO.Directory.CreateDirectory(parent); + } + System.IO.File.WriteAllBytes(fullPath, Encoding.UTF8.GetBytes(content)); + } + + [StarlarkMethod("read_path", Doc = "Read the content of path as UTF-8")] + public string ReadPath( + [Param(Name = "path", Doc = "The Path to read from")] CheckoutPath path) => + Encoding.UTF8.GetString(System.IO.File.ReadAllBytes(AsCheckoutPath(path))); + + [StarlarkMethod("set_executable", Doc = "Set the executable permission of a file")] + public void SetExecutable( + [Param(Name = "path", Doc = "The Path to set the executable permission of")] CheckoutPath path, + [Param(Name = "value", Doc = "Whether or not the file should be executable")] bool value) + { + string full = AsCheckoutPath(path); + if (!OperatingSystem.IsWindows()) + { + var mode = System.IO.File.GetUnixFileMode(full); + var exec = UnixFileMode.UserExecute | UnixFileMode.GroupExecute | UnixFileMode.OtherExecute; + mode = value ? mode | exec : mode & ~exec; + System.IO.File.SetUnixFileMode(full, mode); + } + } + + /// + /// The path containing the repository state to transform. Transformation should be done in-place. + /// + public string GetCheckoutDir() => _checkoutDir; + + [StarlarkMethod("list", Doc = "List files in the checkout/work directory that matches a glob")] + public StarlarkList List( + [Param(Name = "paths", Doc = "A glob representing the paths to list")] Glob glob) + { + var pathMatcher = glob.RelativeTo(_checkoutDir); + var result = new List(); + foreach (var full in System.IO.Directory.EnumerateFiles( + _checkoutDir, "*", System.IO.SearchOption.AllDirectories)) + { + string normalized = full.Replace('\\', '/'); + if (pathMatcher.Matches(normalized)) + { + result.Add(new CheckoutPath(PathOps.Relativize(_checkoutDir, normalized), _checkoutDir)); + } + } + return StarlarkList.ImmutableCopyOf(result.Cast()); + } + + private string AsCheckoutPath(CheckoutPath path) + { + string resolved = PathOps.Resolve(_checkoutDir, path.GetPath()); + string normalized = PathOps.Normalize(resolved); + if (!PathOps.StartsWith(normalized, _checkoutDir)) + { + throw StarlarkRt.Errorf( + "{0} is not inside the checkout directory or links to a file outside the path." + + " Normalized path was {1}, checkout dir was {2}", + path, normalized, _checkoutDir); + } + return normalized; + } +} diff --git a/src/Copybara.Core/CheckoutPath.cs b/src/Copybara.Core/CheckoutPath.cs new file mode 100644 index 000000000..df1ba385e --- /dev/null +++ b/src/Copybara.Core/CheckoutPath.cs @@ -0,0 +1,275 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara; + +/// +/// Represents a file that is exposed to Skylark. +/// +/// Files are always relative to the checkout dir and normalized. Paths are represented as +/// forward-slash-separated strings, mirroring java.nio.file.Path semantics. +/// +[StarlarkBuiltin("Path", Doc = "Represents a path in the checkout directory")] +public class CheckoutPath : IComparable, IStarlarkPrintableValue +{ + private readonly string _path; + private readonly string _checkoutDir; + + internal CheckoutPath(string path, string checkoutDir) + { + _path = Preconditions.CheckNotNull(path); + _checkoutDir = Preconditions.CheckNotNull(checkoutDir); + } + + private CheckoutPath Create(string path) => CreateWithCheckoutDir(path, _checkoutDir); + + public static CheckoutPath CreateWithCheckoutDir(string relative, string checkoutDir) + { + if (PathOps.IsAbsolute(relative)) + { + throw StarlarkRt.Errorf("Absolute paths are not allowed: {0}", relative); + } + string targetPath = PathOps.Normalize(PathOps.Resolve(checkoutDir, relative)); + if (!PathOps.StartsWith(targetPath, checkoutDir)) + { + throw StarlarkRt.Errorf("Escaping the checkout dir is not allowed: {0}", relative); + } + + return new CheckoutPath(PathOps.Normalize(relative), checkoutDir); + } + + [StarlarkMethod( + "path", + Doc = "Full path relative to the checkout directory", + StructField = true)] + public string PathAsString() => _path; + + /// + /// The full path pointing to the real location of the checkout file. Use only for internal + /// implementations and do not make this available to Starlark. + /// + public string FullPath() => PathOps.Resolve(_checkoutDir, _path); + + [StarlarkMethod( + "name", + Doc = "Filename of the path. For foo/bar/baz.txt it would be baz.txt", + StructField = true)] + public string Name() => PathOps.GetFileName(_path); + + [StarlarkMethod("parent", Doc = "Get the parent path", StructField = true)] + public object Parent() + { + string? parent = PathOps.GetParent(_path); + if (parent == null) + { + // nio equivalent of new_path("foo").parent returns null, but we want to be able to do + // foo.parent.resolve("bar"). + return _path.Length == 0 ? StarlarkRt.None : Create(""); + } + return Create(parent); + } + + [StarlarkMethod( + "relativize", + Doc = + "Constructs a relative path between this path and a given path. For example:
" + + " path('a/b').relativize('a/b/c/d')
" + + "returns 'c/d'")] + public CheckoutPath Relativize( + [Param(Name = "other", Doc = "The path to relativize against this path")] CheckoutPath other) => + Create(PathOps.Relativize(_path, other._path)); + + [StarlarkMethod( + "resolve", + Doc = "Resolve the given path against this path.")] + public CheckoutPath Resolve( + [Param( + Name = "child", + AllowedTypes = new[] { typeof(string), typeof(CheckoutPath) }, + Doc = "Resolve the given path against this path. The parameter" + + " can be a string or a Path.")] + object child) + { + if (child is string s) + { + return Create(PathOps.Resolve(_path, s)); + } + if (child is CheckoutPath cp) + { + return Create(PathOps.Resolve(_path, cp._path)); + } + throw StarlarkRt.Errorf("Cannot resolve children for type {0}: {1}", child.GetType().Name, child); + } + + [StarlarkMethod( + "resolve_sibling", + Doc = "Resolve the given path against this path.")] + public CheckoutPath ResolveSibling( + [Param( + Name = "other", + AllowedTypes = new[] { typeof(string), typeof(CheckoutPath) }, + Doc = "Resolve the given path against this path. The parameter can be a string or a Path.")] + object other) + { + if (other is string s) + { + return Create(PathOps.ResolveSibling(_path, s)); + } + if (other is CheckoutPath cp) + { + return Create(PathOps.ResolveSibling(_path, cp._path)); + } + throw StarlarkRt.Errorf("Cannot resolve sibling for type {0}: {1}", other.GetType().Name, other); + } + + [StarlarkMethod("attr", Doc = "Get the file attributes, for example size.", StructField = true)] + public CheckoutPathAttributes Attr() + { + try + { + string full = PathOps.Resolve(_checkoutDir, _path); + bool isSymlink = + (System.IO.File.GetAttributes(full) & System.IO.FileAttributes.ReparsePoint) != 0; + long size = isSymlink ? 0 : new System.IO.FileInfo(full).Length; + return new CheckoutPathAttributes(_path, size, isSymlink); + } + catch (System.IO.IOException e) + { + throw StarlarkRt.Errorf("Error getting attributes for {0}:{1}", _path, e); + } + } + + [StarlarkMethod("read_symlink", Doc = "Read the symlink")] + public CheckoutPath ReadSymbolicLink() + { + try + { + string symlinkPath = PathOps.Resolve(_checkoutDir, _path); + var info = new System.IO.FileInfo(symlinkPath); + if ((System.IO.File.GetAttributes(symlinkPath) & System.IO.FileAttributes.ReparsePoint) == 0) + { + throw StarlarkRt.Errorf("{0} is not a symlink", _path); + } + + var resolvedSymlink = + FileUtil.ResolveSymlink(Glob.AllFiles.RelativeTo(_checkoutDir), symlinkPath); + if (resolvedSymlink.TargetLocationValue != FileUtil.ResolvedSymlink.TargetLocation.Inside) + { + throw StarlarkRt.Errorf( + "Symlink {0} does not point to a file inside the checkout dir: {1}", + symlinkPath, resolvedSymlink.RegularFile); + } + + return Create(PathOps.Relativize(_checkoutDir, resolvedSymlink.RegularFile)); + } + catch (System.IO.IOException e) + { + throw StarlarkRt.Errorf("Cannot resolve symlink {0}: {1}", _path, e); + } + } + + [StarlarkMethod("remove", Doc = "Delete self")] + public void Remove() + { + try + { + System.IO.File.Delete(PathOps.Resolve(_checkoutDir, _path)); + } + catch (System.IO.FileNotFoundException e) + { + throw StarlarkRt.Errorf("Could not find file {0}, received error {1}", _path, e.ToString()); + } + catch (System.IO.IOException e) + { + throw new ValidationException("Could not delete file for unknown reason", e); + } + } + + [StarlarkMethod( + "rmdir", + Doc = + "Delete all files in a directory. If recursive is true, delete descendants of all files" + + " in directory")] + public void RmDir( + [Param( + Name = "recursive", + Named = true, + Doc = "When true, delete descendants of self and of siblings", + DefaultValue = "False")] + bool recursive) + { + try + { + string full = PathOps.Resolve(_checkoutDir, _path); + if (!System.IO.Directory.Exists(full) && !System.IO.File.Exists(full)) + { + return; + } + if (recursive) + { + FileUtil.DeleteRecursively(full); + } + else + { + System.IO.Directory.Delete(full); + } + } + catch (System.IO.IOException e) + { + throw new ValidationException("Could not delete file for unknown reason", e); + } + } + + [StarlarkMethod("exists", Doc = "Check whether a file, directory or symlink exists at this path")] + public bool FileExists() + { + string full = PathOps.Resolve(_checkoutDir, _path); + return System.IO.File.Exists(full) || System.IO.Directory.Exists(full); + } + + public string GetPath() => _path; + + public string GetCheckoutDir() => _checkoutDir; + + public override string ToString() => _path; + + public int CompareTo(CheckoutPath? o) => + string.CompareOrdinal(_path, o?._path); + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(_path); + + public override int GetHashCode() => _path.GetHashCode(); + + public override bool Equals(object? o) + { + if (ReferenceEquals(this, o)) + { + return true; + } + if (o is not CheckoutPath other) + { + return false; + } + return string.Equals(_path, other._path); + } +} diff --git a/src/Copybara.Core/CheckoutPathAttributes.cs b/src/Copybara.Core/CheckoutPathAttributes.cs new file mode 100644 index 000000000..3423648f0 --- /dev/null +++ b/src/Copybara.Core/CheckoutPathAttributes.cs @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara; + +/// Represents file attributes exposed to Skylark. +[StarlarkBuiltin("PathAttributes", Doc = "Represents a path attributes like size.")] +public class CheckoutPathAttributes : IStarlarkValue +{ + private readonly string _path; + private readonly long _size; + private readonly bool _isSymlink; + + internal CheckoutPathAttributes(string path, long size, bool isSymlink) + { + _path = Preconditions.CheckNotNull(path); + _size = size; + _isSymlink = isSymlink; + } + + [StarlarkMethod( + "size", + Doc = "The size of the file. Throws an error if file size > 2GB.", + StructField = true)] + public int Size() + { + if (_size is > int.MaxValue or < int.MinValue) + { + throw StarlarkRt.Errorf( + "File {0} is too big to compute the size: {1} bytes", _path, _size); + } + return (int)_size; + } + + [StarlarkMethod("symlink", Doc = "Returns true if it is a symlink", StructField = true)] + public bool IsSymlink() => _isSymlink; +} diff --git a/src/Copybara.Core/Checks/ApiChecker.cs b/src/Copybara.Core/Checks/ApiChecker.cs new file mode 100644 index 000000000..9487c3a4a --- /dev/null +++ b/src/Copybara.Core/Checks/ApiChecker.cs @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Checks; + +/// +/// A checker for API clients that delegates on a and provides convenience +/// methods for checking one or more pairs of field names and values, plus error handling. +/// +public class ApiChecker +{ + private readonly IChecker _checker; + private readonly Console _console; + + public ApiChecker(IChecker checker, Console console) + { + _checker = Preconditions.CheckNotNull(checker); + _console = Preconditions.CheckNotNull(console); + } + + /// Performs a check on the given request field. + /// + public void Check(string field, object value) => + DoCheck(ImmutableDictionary.CreateRange(new[] + { + KeyValuePair.Create(field, value.ToString()!), + })); + + /// Performs a check on the given request fields. + /// + public void Check(string field1, object value1, string field2, object value2) => + DoCheck(ImmutableDictionary.CreateRange(new[] + { + KeyValuePair.Create(field1, value1.ToString()!), + KeyValuePair.Create(field2, value2.ToString()!), + })); + + /// Performs a check on the given request fields. + /// + public void Check( + string field1, object value1, string field2, object value2, string field3, object value3) => + DoCheck(ImmutableDictionary.CreateRange(new[] + { + KeyValuePair.Create(field1, value1.ToString()!), + KeyValuePair.Create(field2, value2.ToString()!), + KeyValuePair.Create(field3, value3.ToString()!), + })); + + private void DoCheck(ImmutableDictionary data) + { + try + { + _checker.DoCheck(data, _console); + } + catch (IOException e) + { + throw new InvalidOperationException("Error running checker", e); + } + } +} diff --git a/src/Copybara.Core/Checks/Checker.cs b/src/Copybara.Core/Checks/Checker.cs new file mode 100644 index 000000000..c6c8a9688 --- /dev/null +++ b/src/Copybara.Core/Checks/Checker.cs @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Starlark.Annot; +using Starlark.Eval; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Checks; + +// TODO(port): minimal port of com.google.copybara.checks.Checker, created here because the http +// package depends on it. Consolidate if the checks package gets a fuller port. + +/// A generic interface for performing checks on string contents and files. +[StarlarkBuiltin("checker", Doc = "A checker to be run on arbitrary data and files")] +public interface IChecker : IStarlarkValue +{ + /// Performs a check on the given contents. + /// if the check produced errors. + void DoCheck(ImmutableDictionary fields, Console console); + + /// Performs a check on the files inside a given path. + /// if the check produced errors. + void DoCheck(string target, Console console); +} diff --git a/src/Copybara.Core/Checks/CheckerException.cs b/src/Copybara.Core/Checks/CheckerException.cs new file mode 100644 index 000000000..aa5661ccd --- /dev/null +++ b/src/Copybara.Core/Checks/CheckerException.cs @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Exceptions; + +namespace Copybara.Checks; + +// TODO(port): minimal port of com.google.copybara.checks.CheckerException. + +/// Indicates an exception running a . +public class CheckerException : ValidationException +{ + public CheckerException(string message) + : base(message) + { + } + + public CheckerException(string message, Exception? cause) + : base(message, cause) + { + } +} diff --git a/src/Copybara.Core/Config/CapturingConfigFile.cs b/src/Copybara.Core/Config/CapturingConfigFile.cs new file mode 100644 index 000000000..e36040d22 --- /dev/null +++ b/src/Copybara.Core/Config/CapturingConfigFile.cs @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; + +namespace Copybara.Config; + +/// +/// A config file that records the children created from it. Useful for collecting dependencies in +/// dry runs. +/// +internal sealed class CapturingConfigFile : ConfigFile +{ + private readonly LinkedHashSet _children = new(); + private readonly ConfigFile _wrapped; + + internal CapturingConfigFile(ConfigFile config) => _wrapped = Preconditions.CheckNotNull(config); + + public ConfigFile Resolve(string path) + { + var resolved = new CapturingConfigFile(_wrapped.Resolve(path)); + _children.Add(resolved); + return resolved; + } + + public ImmutableDictionary ResolveAll(IReadOnlySet paths) + { + var result = ImmutableDictionary.CreateBuilder(); + foreach (var e in _wrapped.ResolveAll(paths)) + { + var capturingConfigFile = new CapturingConfigFile(e.Value); + result[e.Key] = capturingConfigFile; + _children.Add(capturingConfigFile); + } + return result.ToImmutable(); + } + + public string Path() => _wrapped.Path(); + + public byte[] ReadContentBytes() => _wrapped.ReadContentBytes(); + + public string GetIdentifier() => _wrapped.GetIdentifier(); + + /// + /// Retrieve collected dependencies. + /// + /// A map from path to the wrapped ConfigFile for each ConfigFile created by this or one + /// of its descendants. Includes this. + internal ImmutableDictionary GetAllLoadedFiles() + { + var map = new Dictionary(); + GetAllLoadedFiles(map); + return map.ToImmutableDictionary(); + } + + private void GetAllLoadedFiles(Dictionary map) + { + map[Path()] = _wrapped; + foreach (var child in _children) + { + child.GetAllLoadedFiles(map); + } + } + + public override bool Equals(object? otherObject) => + otherObject is CapturingConfigFile other + && other._wrapped.Equals(_wrapped) + && _children.SetEquals(other._children); + + public override int GetHashCode() => Path().GetHashCode(); + + public override string ToString() => + $"CapturingConfigFile{{children={_children.Count}, wrapped={_wrapped}}}"; + + /// An insertion-ordered set, mirroring Java's LinkedHashSet semantics. + private sealed class LinkedHashSet : IEnumerable + where T : notnull + { + private readonly Dictionary _index = new(); + private readonly List _items = new(); + + public void Add(T item) + { + if (_index.TryAdd(item, _items.Count)) + { + _items.Add(item); + } + } + + public int Count => _items.Count; + + public bool SetEquals(LinkedHashSet other) => + _index.Count == other._index.Count && _index.Keys.All(other._index.ContainsKey); + + public IEnumerator GetEnumerator() => _items.GetEnumerator(); + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => + GetEnumerator(); + } +} diff --git a/src/Copybara.Core/Config/Config.cs b/src/Copybara.Core/Config/Config.cs new file mode 100644 index 000000000..0626bd90c --- /dev/null +++ b/src/Copybara.Core/Config/Config.cs @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Exceptions; + +namespace Copybara.Config; + +/// +/// Configuration for a Copybara project. +/// +/// Objects of this class represent a parsed Copybara configuration. +/// +public sealed class Config +{ + private readonly ImmutableDictionary _migrations; + private readonly string _location; + private readonly ImmutableDictionary _globals; + + public Config( + IReadOnlyDictionary migrations, + string location, + IReadOnlyDictionary globals) + { + _migrations = migrations.ToImmutableDictionary(); + _location = Preconditions.CheckNotNull(location); + _globals = globals.ToImmutableDictionary(); + } + + /// Returns the named after . + /// if no migration with the given name exists + public IMigration GetMigration(string migrationName) + { + ValidationException.CheckCondition( + _migrations.ContainsKey(migrationName), + "No migration with name '{0}' exists. Valid migrations: {1}", + migrationName, + string.Join(", ", _migrations.Keys)); + return _migrations[migrationName]; + } + + /// + /// Location of the top-level config file. An arbitrary string meant to be used for + /// logging/debugging. It shouldn't be parsed, as the format might change. + /// + public string GetLocation() => _location; + + /// + /// Reads values from the global frame of the skylark environment, i.e. global variables. + /// + public T? GetGlobalEnvironmentVariable(string name) + where T : class => + _globals.TryGetValue(name, out object? value) ? value as T : null; + + /// Returns all the migrations in this configuration. + public ImmutableDictionary GetMigrations() => _migrations; + + public override bool Equals(object? o) + { + if (ReferenceEquals(this, o)) + { + return true; + } + if (o is not Config config) + { + return false; + } + return DictionaryEquals(_migrations, config._migrations); + } + + public override int GetHashCode() + { + int hash = 17; + foreach (var e in _migrations) + { + hash ^= e.Key.GetHashCode(); + } + return hash; + } + + public override string ToString() => + $"Config{{migrations=[{string.Join(", ", _migrations.Keys)}], location={_location}}}"; + + private static bool DictionaryEquals( + ImmutableDictionary a, ImmutableDictionary b) + { + if (a.Count != b.Count) + { + return false; + } + foreach (var e in a) + { + if (!b.TryGetValue(e.Key, out IMigration? other) || !Equals(e.Value, other)) + { + return false; + } + } + return true; + } +} diff --git a/src/Copybara.Core/Config/ConfigFile.cs b/src/Copybara.Core/Config/ConfigFile.cs new file mode 100644 index 000000000..84622419e --- /dev/null +++ b/src/Copybara.Core/Config/ConfigFile.cs @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text; +using Copybara.Exceptions; +using Copybara.Util; + +namespace Copybara.Config; + +/// +/// An object representing a configuration file and that it can be used to resolve +/// other config files relative to this one. +/// +public interface ConfigFile +{ + /// + /// Check if the path is absolute and validates that the path is normalized. + /// + /// if the path is not normalized + static bool IsAbsolute(string path) + { + bool isAbsolute = path.StartsWith("//", StringComparison.Ordinal); + // Remove '//' for absolute paths + string withoutPrefix = isAbsolute ? path.Substring(2) : path; + try + { + FileUtil.CheckNormalizedRelative(withoutPrefix); + return isAbsolute; + } + catch (ArgumentException e) + { + throw new CannotResolveLabel( + string.Format("Invalid path '{0}': {1}", withoutPrefix, e.Message)); + } + } + + /// + /// Resolve relative to the current config file. + /// + /// if the path cannot be resolved to a content + ConfigFile Resolve(string path); + + /// + /// Resolve a set of configs paths in a batch. This can be used by implementors to check + /// existence or preload the bytes of the content in batch/parallel. + /// + /// a set of paths + /// a map from paths to + /// if any of the paths cannot be resolved + ImmutableDictionary ResolveAll(IReadOnlySet paths) + { + var result = ImmutableDictionary.CreateBuilder(); + foreach (string path in paths) + { + result[path] = Resolve(path); + } + return result.ToImmutable(); + } + + /// Resolved, non-relative name of the config file. + string Path(); + + /// + /// Get the contents of the file. + /// + /// Implementations of this interface should prefer to not eagerly load the content when + /// this method is called in order to allow the callers to check their own cache if they already + /// have . + /// + byte[] ReadContentBytes(); + + /// Utility function to read the content of the config file as String. + string ReadContent() => Encoding.UTF8.GetString(ReadContentBytes()); + + /// + /// Return a string representing a stable identifier that works between different + /// implementations. Note that this is best effort based on several + /// heuristics. + /// + /// If root is not defined or cannot be computed, it will return the absolute path. + /// + /// Users of this method should not try to parse the string, since it is subject to + /// change. + /// + string GetIdentifier(); +} diff --git a/src/Copybara.Core/Config/ConfigValidator.cs b/src/Copybara.Core/Config/ConfigValidator.cs new file mode 100644 index 000000000..6b4c9b1e7 --- /dev/null +++ b/src/Copybara.Core/Config/ConfigValidator.cs @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Config; + +/// +/// Validates Copybara s and returns a . +/// +/// Implementations of this interface should not throw exceptions for validation errors. +/// +public interface ConfigValidator +{ + ValidationResult Validate(Config config, string migrationName) + { + var resultBuilder = new ValidationResult.Builder(); + CheckAtLeastOneMigration(resultBuilder, config); + return resultBuilder.Build(); + } + + void CheckAtLeastOneMigration(ValidationResult.Builder resultBuilder, Config config) + { + if (config.GetMigrations().Count == 0) + { + resultBuilder.Error("At least one migration is required."); + } + } +} diff --git a/src/Copybara.Core/Config/ConfigWithDependencies.cs b/src/Copybara.Core/Config/ConfigWithDependencies.cs new file mode 100644 index 000000000..3c56602ba --- /dev/null +++ b/src/Copybara.Core/Config/ConfigWithDependencies.cs @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; + +namespace Copybara.Config; + +/// +/// A class that contains a loaded config and all the config files that were accessed during the +/// parsing. +/// +/// Upstream this is a nested class SkylarkParser.ConfigWithDependencies; it is promoted +/// to a top-level type in the .NET port. +/// +public sealed class ConfigWithDependencies +{ + private readonly ImmutableDictionary _files; + private readonly Config _config; + + internal ConfigWithDependencies(ImmutableDictionary files, Config config) + { + _config = config; + _files = files; + } + + public Config GetConfig() => _config; + + public ImmutableDictionary GetFiles() => _files; +} diff --git a/src/Copybara.Core/Config/GlobalMigrations.cs b/src/Copybara.Core/Config/GlobalMigrations.cs new file mode 100644 index 000000000..ac0c81261 --- /dev/null +++ b/src/Copybara.Core/Config/GlobalMigrations.cs @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.RegularExpressions; +using Copybara.Common; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Config; + +/// Global variable that holds the registered migrations in the config files. +[StarlarkBuiltin(GlobalMigrations.GLOBAL_MIGRATIONS, + Doc = "Global variable that holds the registered migrations in the config files", + Documented = false)] +public sealed class GlobalMigrations : IStarlarkValue +{ + private static readonly Regex MigrationNameFormat = + new(@"^[a-zA-Z0-9_\-\./]+$", RegexOptions.Compiled); + + private const string MigrationNamePattern = "[a-zA-Z0-9_\\-\\./]+"; + + internal const string GLOBAL_MIGRATIONS = "global_migrations"; + + private readonly Dictionary _migrations = new(); + + public static GlobalMigrations GetGlobalMigrations(Module module) => + (GlobalMigrations)Preconditions.CheckNotNull(module.GetPredeclared(GLOBAL_MIGRATIONS)); + + public IReadOnlyDictionary GetMigrations() => _migrations; + + /// if a migration with the name already exists or the name is + /// invalid + public void AddMigration(string name, IMigration migration) + { + CheckMigrationName(name); + SkylarkUtil.Check( + _migrations.TryAdd(name, migration), + "A migration with the name '{0}' is already defined", + name); + } + + /// Checks if a migration name conforms to the expected format. + /// Migration name + /// If the name does not conform to the expected format + public static void CheckMigrationName(string name) + { + SkylarkUtil.Check( + MigrationNameFormat.IsMatch(name), + "Migration name '{0}' doesn't conform to expected pattern: {1}", + name, + MigrationNamePattern); + } +} diff --git a/src/Copybara.Core/Config/ILabelsAwareModule.cs b/src/Copybara.Core/Config/ILabelsAwareModule.cs new file mode 100644 index 000000000..fdcf8f831 --- /dev/null +++ b/src/Copybara.Core/Config/ILabelsAwareModule.cs @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Starlark.Eval; + +namespace Copybara.Config; + +/// +/// A StarlarkBuiltin that implements this interface will be given information about the config files +/// and resources loaded by the configuration. +/// +public interface ILabelsAwareModule +{ + /// + /// Called before invoking any methods on a module in order to give the module access to the + /// current config file. This may be called multiple times, in which case only the most recent + /// should be used. + /// + void SetConfigFile(ConfigFile mainConfigFile, ConfigFile currentConfigFile) + { + } + + /// + /// Called before invoking any methods on a module to give the module access to the current + /// workflow name. This may be called multiple times, in which case only the most recent should + /// be used. + /// + void SetWorkflowName(string workflowName) + { + } + + /// + /// A supplier that returns all the files loaded by the configuration loading. The supplier + /// shouldn't be evaluated before loading finishes. + /// + void SetAllConfigResources(Func> configs) + { + } + + /// + /// Set handler for print statements executed by Starlark code run during a migration (for + /// example dynamic transformations, migration hooks or feedback mechanism). + /// + void SetPrintHandler(StarlarkThread.PrintHandler printHandler) + { + } +} diff --git a/src/Copybara.Core/Config/IMigration.cs b/src/Copybara.Core/Config/IMigration.cs new file mode 100644 index 000000000..f1da67e55 --- /dev/null +++ b/src/Copybara.Core/Config/IMigration.cs @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Revision; +using Starlark.Eval; + +namespace Copybara.Config; + +/// +/// A migration is a process that moves files and/or metadata (comments, labels...) at a particular +/// revision from one/many systems to one/many destinations. +/// +/// For helping with the migration a working directory is provided to do any temporary file +/// operations. +/// +public interface IMigration +{ + /// + /// Run a migration for a list of source references. If empty, the default (if any) will be used. + /// + /// Different implementations of Migration might process the list of source references + /// differently (batching them, or running one by one). + /// + /// a working directory for doing file operations if needed. + /// the source references to be migrated. If not present the default + /// (if any) for the migration will be used. + void Run(string workdir, IReadOnlyList sourceRefs); + + Info GetInfo() => Info.Empty; + + /// The migration's name. + string GetName(); + + /// An optional description that users can set to describe what this workflow achieves. + string? GetDescription(); + + /// The migration's mode. + string GetModeString(); + + /// The migration's main config file. + ConfigFile GetMainConfigFile(); + + /// + /// Returns a multimap containing enough data to fingerprint the origin for validation purposes. + /// + Common.ImmutableListMultimap GetOriginDescription(); + + /// + /// Returns a multimap containing enough data to fingerprint the destination for validation + /// purposes. + /// + Common.ImmutableListMultimap GetDestinationDescription(); + + /// Returns a multimap containing enough data to fingerprint credentials used. + IReadOnlyList> GetCredentialDescription(); + + /// Returns the Starlark call stack captured when the migration was defined. + ImmutableArray GetDefinitionStack() => + ImmutableArray.Empty; +} diff --git a/src/Copybara.Core/Config/IOptionsAwareModule.cs b/src/Copybara.Core/Config/IOptionsAwareModule.cs new file mode 100644 index 000000000..23da2c451 --- /dev/null +++ b/src/Copybara.Core/Config/IOptionsAwareModule.cs @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Config; + +/// +/// A StarlarkBuiltin that implements this interface will be initialized with the options. +/// +/// This method will be invoked just after registering the namespace objects in Skylark. +/// +public interface IOptionsAwareModule +{ + /// Set the options for the current Copybara run. + void SetOptions(Options options); +} diff --git a/src/Copybara.Core/Config/MapConfigFile.cs b/src/Copybara.Core/Config/MapConfigFile.cs new file mode 100644 index 000000000..70ef0f7b2 --- /dev/null +++ b/src/Copybara.Core/Config/MapConfigFile.cs @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Exceptions; + +namespace Copybara.Config; + +/// +/// A Config file implementation that uses a map for storing the internal data structure. +/// +/// Assumes all paths to be absolute. +/// +public class MapConfigFile : ConfigFile +{ + private readonly ImmutableDictionary _configFiles; + private readonly string _current; + + public MapConfigFile(ImmutableDictionary configFiles, string current) + { + _configFiles = configFiles; + _current = current; + } + + public ConfigFile Resolve(string path) + { + string resolved = ConfigFile.IsAbsolute(path) + ? ContainsLabel(path.Substring(2)) + : RelativeToCurrentPath(path); + if (!_configFiles.ContainsKey(resolved)) + { + throw new CannotResolveLabel( + string.Format("Cannot resolve '{0}': '{1}' does not exist.", path, resolved)); + } + return new MapConfigFile(_configFiles, resolved); + } + + public string Path() => _current; + + public string GetIdentifier() => Path(); + + public byte[] ReadContentBytes() => _configFiles[_current]; + + public override string ToString() => + $"MapConfigFile{{current={_current}, configFiles=[{string.Join(", ", _configFiles.Keys)}]}}"; + + private string RelativeToCurrentPath(string label) + { + int i = _current.LastIndexOf('/'); + string resolved = i == -1 ? label : _current.Substring(0, i) + "/" + label; + return ContainsLabel(resolved); + } + + private string ContainsLabel(string resolved) + { + if (!_configFiles.ContainsKey(resolved)) + { + throw new CannotResolveLabel( + string.Format("Cannot resolve '{0}': does not exist.", resolved)); + } + return resolved; + } +} diff --git a/src/Copybara.Core/Config/MigrationValidator.cs b/src/Copybara.Core/Config/MigrationValidator.cs new file mode 100644 index 000000000..bd244d012 --- /dev/null +++ b/src/Copybara.Core/Config/MigrationValidator.cs @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Config; + +/// +/// Validates Copybara s and returns a . +/// +/// Implementations of this interface should not throw exceptions for validation errors. +/// +// Reference-forward: upstream dispatches on Workflow, Mirror and ActionMigration. Only +// ActionMigration is ported so far; Workflow/Mirror dispatch is added during final consolidation +// once those types land (Copybara.Workflow, Copybara.Git.Mirror). +public abstract class MigrationValidator +{ + public ValidationResult Validate(IMigration migration, Config config) + { + if (migration is ActionMigration actionMigration) + { + return ValidateActionMigration(migration.GetName(), actionMigration, config); + } + // TODO(consolidation): add Workflow and Mirror dispatch when those types are ported. + throw new InvalidOperationException($"Validation missing for {migration}"); + } + + /// Performs specific validation of an migration. + protected abstract ValidationResult ValidateActionMigration( + string name, ActionMigration actionMigration, Config config); +} diff --git a/src/Copybara.Core/Config/PathBasedConfigFile.cs b/src/Copybara.Core/Config/PathBasedConfigFile.cs new file mode 100644 index 000000000..620d22524 --- /dev/null +++ b/src/Copybara.Core/Config/PathBasedConfigFile.cs @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Copybara.Exceptions; +using SysPath = System.IO.Path; + +namespace Copybara.Config; + +/// +/// A Skylark dependency resolver that resolves relative paths and absolute paths if +/// rootPath is defined. +/// +public class PathBasedConfigFile : ConfigFile +{ + private readonly string _path; + private readonly string? _rootPath; + private readonly string? _identifierPrefix; + + public PathBasedConfigFile(string path, string? rootPath, string? identifierPrefix) + { + Preconditions.CheckArgument(SysPath.IsPathRooted(path), "path must be absolute"); + _path = path; + _rootPath = rootPath; + _identifierPrefix = identifierPrefix; + if (identifierPrefix != null) + { + // Check we don't generate weird identifiers like identifierPrefix + "/absolute/path" + Preconditions.CheckNotNull(rootPath, "identifierPrefix requires a non null root"); + } + } + + public ConfigFile Resolve(string path) + { + string resolved = ConfigFile.IsAbsolute(path) + ? RelativeToRoot(path) + : RelativeToCurrentPath(path); + + if (!File.Exists(resolved) && !Directory.Exists(resolved)) + { + throw new CannotResolveLabel( + string.Format("Cannot find '{0}'. '{1}' does not exist.", path, resolved)); + } + if (!File.Exists(resolved)) + { + throw new CannotResolveLabel( + string.Format("Cannot find '{0}'. '{1}' is not a file.", path, resolved)); + } + return new PathBasedConfigFile(resolved, _rootPath, _identifierPrefix); + } + + public string Path() => _path; + + public string GetIdentifier() + { + if (_rootPath == null) + { + return Path(); + } + + return (string.IsNullOrEmpty(_identifierPrefix) ? "" : _identifierPrefix + "/") + + Relativize(_rootPath, _path); + } + + private string RelativeToCurrentPath(string label) + { + string? dir = SysPath.GetDirectoryName(_path); + return dir == null ? label : SysPath.Combine(dir, label); + } + + private string RelativeToRoot(string path) + { + if (_rootPath == null) + { + throw new CannotResolveLabel( + "Absolute paths are not allowed because the root config path couldn't be" + + " automatically detected. Use " + GeneralOptions.ConfigRootFlag); + } + return SysPath.Combine(_rootPath, path.Substring(2)); + } + + public byte[] ReadContentBytes() + { + try + { + return File.ReadAllBytes(_path); + } + catch (FileNotFoundException e) + { + throw new CannotResolveLabel("Cannot resolve " + _path, e); + } + catch (DirectoryNotFoundException e) + { + throw new CannotResolveLabel("Cannot resolve " + _path, e); + } + } + + public override string ToString() => + $"PathBasedConfigFile{{path={_path}, rootPath={_rootPath}, identifierPrefix={_identifierPrefix}}}"; + + private static string Relativize(string root, string path) + { + string rel = SysPath.GetRelativePath(root, path); + // Normalize to forward slashes so identifiers are stable across platforms. + return rel.Replace(SysPath.DirectorySeparatorChar, '/'); + } +} diff --git a/src/Copybara.Core/Config/ResolveDelegateConfigFile.cs b/src/Copybara.Core/Config/ResolveDelegateConfigFile.cs new file mode 100644 index 000000000..a3140aec2 --- /dev/null +++ b/src/Copybara.Core/Config/ResolveDelegateConfigFile.cs @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Copybara.Exceptions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Copybara.Config; + +/// +/// A that delegates to a main config file and falls back to a secondary one +/// for file resolution, if necessary. +/// +/// This is useful for cases where generated in-memory configurations have dependencies on +/// persisted configurations. +/// +public sealed class ResolveDelegateConfigFile : ConfigFile +{ + private static readonly ILogger Logger = NullLogger.Instance; + + private readonly ConfigFile _mainConfigFile; + private readonly ConfigFile _secondConfigFile; + + public ResolveDelegateConfigFile(ConfigFile mainConfigFile, ConfigFile secondConfigFile) + { + _mainConfigFile = Preconditions.CheckNotNull(mainConfigFile); + _secondConfigFile = Preconditions.CheckNotNull(secondConfigFile); + } + + public ConfigFile Resolve(string path) + { + try + { + return _mainConfigFile.Resolve(path); + } + catch (CannotResolveLabel) + { + Logger.LogInformation( + "Could not resolve {Path} from {Main}. Resolving from {Second}.", + path, _mainConfigFile.Path(), _secondConfigFile.Path()); + try + { + return _secondConfigFile.Resolve(path); + } + catch (CannotResolveLabel crl) + { + throw new CannotResolveLabel( + string.Format( + "Could not resolve main config or second config to path '{0}'. Main config" + + " path is '{1}', second config path is '{2}'", + path, _mainConfigFile.Path(), _secondConfigFile.Path()), + crl); + } + } + } + + public string Path() => _mainConfigFile.Path(); + + public byte[] ReadContentBytes() => _mainConfigFile.ReadContentBytes(); + + public string GetIdentifier() => _mainConfigFile.GetIdentifier(); +} diff --git a/src/Copybara.Core/Config/SkylarkParser.cs b/src/Copybara.Core/Config/SkylarkParser.cs new file mode 100644 index 000000000..e10f6e823 --- /dev/null +++ b/src/Copybara.Core/Config/SkylarkParser.cs @@ -0,0 +1,371 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Reflection; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Util.Console; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Starlark.Annot; +using Starlark.Eval; +using Starlark.Syntax; +using Console = Copybara.Util.Console.Console; +using StarlarkRt = Starlark.Eval.Starlark; +using Module = Starlark.Eval.Module; +using Tuple = Starlark.Eval.Tuple; +using FileOptions = Starlark.Syntax.FileOptions; + +namespace Copybara.Config; + +/// Loads Copybara configs out of Skylark files. +public class SkylarkParser +{ + private static readonly ILogger Logger = NullLogger.Instance; + private const string DefaultExtension = ".bara.sky"; + + private static readonly ImmutableHashSet AllowedLoadExtensions = + ImmutableHashSet.Create(DefaultExtension, ".scl"); + + private static readonly object VisibilityFunc = new VisibilityFunction(); + + // For now all the modules are namespaces. We don't use variables except for 'core'. + private readonly IReadOnlyList _modules; + private readonly StarlarkMode _validation; + + public SkylarkParser(IReadOnlySet staticModules, StarlarkMode validation) + { + var builder = ImmutableArray.CreateBuilder(); + builder.Add(typeof(GlobalMigrations)); + builder.AddRange(staticModules); + _modules = builder.ToImmutable(); + _validation = validation; + } + + /// + public Config LoadConfig(ConfigFile config, ModuleSet moduleSet, Console console) => + GetConfigWithTransitiveImports(config, moduleSet, console).GetConfig(); + + private Config LoadConfigInternal( + ConfigFile content, + ModuleSet moduleSet, + Func> configFilesSupplier, + Console console) + { + Module module = new Evaluator(this, moduleSet, content, configFilesSupplier, console) + .Eval(content); + GlobalMigrations globalMigrations = GlobalMigrations.GetGlobalMigrations(module); + return new Config( + globalMigrations.GetMigrations(), content.Path(), module.GetPredeclaredBindings()); + } + + public Module ExecuteSkylark(ConfigFile content, ModuleSet moduleSet, Console console) + { + var capturingConfigFile = new CapturingConfigFile(content); + var configFilesSupplier = new ConfigFilesSupplier(); + + Module module = new Evaluator(this, moduleSet, content, configFilesSupplier.Get, console) + .Eval(content); + configFilesSupplier.SetConfigFiles(capturingConfigFile.GetAllLoadedFiles()); + return module; + } + + /// + /// Collect all ConfigFiles retrieved by the parser while loading . + /// + /// Root file of the configuration. + /// the module set providing the Starlark globals. + /// the console to use for printing error/information. + /// A map linking paths to the captured ConfigFiles and the parsed Config. + /// If config is invalid, references an invalid file or + /// contains dependency cycles. + public ConfigWithDependencies GetConfigWithTransitiveImports( + ConfigFile config, ModuleSet moduleSet, Console console) + { + var capturingConfigFile = new CapturingConfigFile(config); + var configFilesSupplier = new ConfigFilesSupplier(); + + Config parsedConfig = + LoadConfigInternal(capturingConfigFile, moduleSet, configFilesSupplier.Get, console); + + ImmutableDictionary allLoadedFiles = + capturingConfigFile.GetAllLoadedFiles(); + + configFilesSupplier.SetConfigFiles(allLoadedFiles); + + return new ConfigWithDependencies(allLoadedFiles, parsedConfig); + } + + private sealed class ConfigFilesSupplier + { + private ImmutableDictionary? _configFiles; + + internal void SetConfigFiles(ImmutableDictionary configFiles) + { + Preconditions.CheckState(_configFiles == null, "Already set"); + _configFiles = Preconditions.CheckNotNull(configFiles); + } + + internal ImmutableDictionary Get() + { + // We need to load all the files before knowing the set of files in the config. + return Preconditions.CheckNotNull( + _configFiles, "Don't call the supplier before loading finishes."); + } + } + + /// A class that traverses and evaluates the config file dependency graph. + private sealed class Evaluator + { + private readonly SkylarkParser _parser; + private readonly HashSet _pending = new(); + private readonly List _pendingOrder = new(); + private readonly Dictionary _loaded = new(); + private readonly Console _console; + private readonly ConfigFile _mainConfigFile; + + // Predeclared environment shared by all files (modules) loaded. + private readonly ImmutableDictionary _environment; + private readonly ModuleSet _moduleSet; + + internal Evaluator( + SkylarkParser parser, + ModuleSet moduleSet, + ConfigFile mainConfigFile, + Func> configFilesSupplier, + Console console) + { + _parser = parser; + _console = Preconditions.CheckNotNull(console); + _mainConfigFile = Preconditions.CheckNotNull(mainConfigFile); + _moduleSet = Preconditions.CheckNotNull(moduleSet); + _environment = _parser.CreateEnvironment(_moduleSet, configFilesSupplier); + } + + internal Module Eval(ConfigFile content) + { + if (_pending.Contains(content.Path())) + { + throw ThrowCycleError(content.Path()); + } + if (_loaded.TryGetValue(content.Path(), out Module? existing)) + { + return existing; + } + _pending.Add(content.Path()); + _pendingOrder.Add(content.Path()); + + // Make the modules available as predeclared bindings. + StarlarkSemantics semantics = StarlarkSemantics.DEFAULT; + Module module = Module.WithPredeclared(semantics, _environment); + + // parse & compile + ParserInput input = ParserInput.FromUTF8(content.ReadContentBytes(), content.Path()); + FileOptions options = + FileOptions.DEFAULT.ToBuilder() + // Ordinarily, load statements should create file-local variables. + // For now, we make them create first-class members of Module.globals. + .LoadBindsGlobally(true) + .AllowToplevelRebinding(true) // allow e.g. x=1; x=2 at top level + .RequireLoadStatementsFirst(_parser._validation == StarlarkMode.Strict) + .Build(); + + Program prog; + try + { + prog = Program.CompileFile( + StarlarkFile.Parse(input, options), StarlarkRt.ModuleAsResolverModule(module)); + } + catch (SyntaxError.Exception ex) + { + foreach (SyntaxError error in ex.Errors) + { + _console.Error(error.ToString()); + } + throw new ValidationException("Error loading config file."); + } + + // process loads + var loadedModules = new Dictionary(); + var fileToLoad = new Dictionary(); + foreach (string l in prog.GetLoads().Distinct()) + { + string key = AllowedLoadExtensions.Any(l.EndsWith) ? l : l + DefaultExtension; + fileToLoad[key] = l; + } + + foreach (var entry in + content + // Resolve all in one call so the implementor can do it in batch/parallel. + .ResolveAll(fileToLoad.Keys.ToImmutableHashSet())) + { + Module loadedModule = Eval(entry.Value); + loadedModules[fileToLoad[entry.Key]] = loadedModule; + } + + // execute + _parser.UpdateEnvironmentForConfigFile( + StarlarkPrint, content, _mainConfigFile, _environment, _moduleSet); + using (Mutability mu = Mutability.Create("CopybaraModules")) + { + StarlarkThread thread = StarlarkThread.CreateTransient(mu, semantics); + thread.SetLoader(m => loadedModules.TryGetValue(m, out Module? md) ? md : null); + thread.SetPrintHandler(StarlarkPrint); + try + { + StarlarkRt.ExecFileProgram(prog, module, thread); + } + catch (EvalException ex) + { + _console.Error(ex.Message); + throw new ValidationException("Error loading config file", ex); + } + } + + _pending.Remove(content.Path()); + _pendingOrder.Remove(content.Path()); + _loaded[content.Path()] = module; + return module; + } + + private void StarlarkPrint(StarlarkThread thread, string msg) => + _console.Verbose(thread.GetCallerLocation() + ": " + msg); + + private ValidationException ThrowCycleError(string cycleElement) + { + var sb = new System.Text.StringBuilder(); + foreach (string element in _pendingOrder) + { + sb.Append(element.Equals(cycleElement) ? "* " : " "); + sb.Append(element).Append('\n'); + } + sb.Append("* ").Append(cycleElement).Append('\n'); + _console.Error("Cycle was detected in the configuration: \n" + sb); + return new ValidationException("Cycle was detected"); + } + } + + /// Updates the module globals with information about the current loaded config file. + private void UpdateEnvironmentForConfigFile( + StarlarkThread.PrintHandler printHandler, + ConfigFile currentConfigFile, + ConfigFile mainConfigFile, + ImmutableDictionary environment, + ModuleSet moduleSet) + { + foreach (object module in moduleSet.GetModules().Values) + { + // We mutate the module per file loaded. Not ideal but it is the best we can do. + if (module is ILabelsAwareModule m) + { + m.SetConfigFile(mainConfigFile, currentConfigFile); + m.SetPrintHandler(printHandler); + } + } + foreach (Type module in _modules) + { + Logger.LogInformation("Creating variable for {Module}", module.FullName); + // We mutate the module per file loaded. Not ideal but it is the best we can do. + if (typeof(ILabelsAwareModule).IsAssignableFrom(module)) + { + var m = (ILabelsAwareModule)environment[GetModuleName(module)]; + m.SetConfigFile(mainConfigFile, currentConfigFile); + m.SetPrintHandler(printHandler); + } + } + } + + /// + /// Create the environment for all evaluations (will be shared between all the dependent files + /// loaded). + /// + private ImmutableDictionary CreateEnvironment( + ModuleSet moduleSet, Func> configFilesSupplier) + { + var env = new Dictionary(); + foreach (var module in moduleSet.GetModules()) + { + Logger.LogInformation("Creating variable for {Module}", module.Key); + if (module.Value is ILabelsAwareModule lam) + { + lam.SetAllConfigResources(configFilesSupplier); + } + // Modules shouldn't use the same name + env[module.Key] = module.Value; + } + + foreach (Type module in _modules) + { + Logger.LogInformation("Creating variable for {Module}", module.FullName); + // Create the module object and associate it with the functions + var envBuilder = new Dictionary(); + var annot = module.GetCustomAttribute(inherit: false); + if (annot != null) + { + envBuilder[annot.Name] = Activator.CreateInstance(module)!; + } + else if (IsLibrary(module)) + { + // Reference-forward: upstream registers @Library modules' methods directly. The + // doc.annotations.Library attribute is not yet ported; handled via IsLibrary probe. + StarlarkRt.AddMethods(envBuilder, Activator.CreateInstance(module)!); + } + + foreach (var e in envBuilder) + { + env[e.Key] = e.Value; + } + + // Add the options to the module that require them + if (typeof(IOptionsAwareModule).IsAssignableFrom(module)) + { + ((IOptionsAwareModule)env[GetModuleName(module)]).SetOptions(moduleSet.GetOptions()); + } + if (typeof(ILabelsAwareModule).IsAssignableFrom(module)) + { + ((ILabelsAwareModule)env[GetModuleName(module)]) + .SetAllConfigResources(configFilesSupplier); + } + } + env["visibility"] = VisibilityFunc; + return env.ToImmutableDictionary(); + } + + private static string GetModuleName(Type cls) => + cls.GetCustomAttribute(inherit: false)!.Name; + + // Probes for a doc.annotations.Library-style attribute by name so the port keeps working once + // that annotation lands, without a hard dependency on it here. + private static bool IsLibrary(Type module) => + module.GetCustomAttributes(inherit: false) + .Any(a => a.GetType().Name is "LibraryAttribute" or "Library"); + + private sealed class VisibilityFunction : IStarlarkCallable + { + public object Call(StarlarkThread thread, Tuple args, Dict kwargs) => StarlarkRt.None; + + public object? Fastcall(StarlarkThread thread, object?[] positional, object?[] named) => + StarlarkRt.None; + + public string Name => "visibility"; + + public void Repr(Printer printer, StarlarkSemantics semantics) => + printer.Append(""); + + public Location Location => Location.BUILTIN; + } +} diff --git a/src/Copybara.Core/Config/SkylarkUtil.cs b/src/Copybara.Core/Config/SkylarkUtil.cs new file mode 100644 index 000000000..f34ec7be1 --- /dev/null +++ b/src/Copybara.Core/Config/SkylarkUtil.cs @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Starlark.Eval; +using Console = Copybara.Util.Console.Console; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara.Config; + +/// +/// Utilities for dealing with Skylark parameter objects and converting them to Java ones. +/// +public static class SkylarkUtil +{ + /// + /// Converts an object that can be the NoneType to the actual object if it is not, or returns the + /// default value if none. + /// + public static T? ConvertFromNoneable(object? obj, T? defaultValue) + { + if (StarlarkRt.IsNullOrNone(obj)) + { + return defaultValue; + } + return (T)obj!; // wildly unsound cast, matching upstream + } + + /// + /// Converts a noneable Starlark object into a nullable reference. If the object is null or None, + /// it will be converted to null. + /// + /// Upstream returns Optional<T>; this port uses nullable per project + /// conventions. + /// + public static T? ConvertToOptional(object? obj) + where T : class => + ConvertFromNoneable(obj, null); + + /// Converts a string to the corresponding enum or fails if invalid value. + /// if the value is not a valid enum member + public static T StringToEnum(string fieldName, string value) + where T : struct, Enum + { + if (Enum.TryParse(value, out T result) && Enum.IsDefined(result)) + { + return result; + } + throw StarlarkRt.Errorf( + "Invalid value '{0}' for field '{1}'. Valid values are: {2}", + value, fieldName, string.Join(", ", Enum.GetNames())); + } + + /// Converts a sequence of strings to a list of the corresponding enum values. + /// if any string cannot be cast to the enum + public static IReadOnlyList StringListToEnumList( + IEnumerable sequence, string fieldName, Console console) + where T : struct, Enum + { + var list = sequence.ToList(); + try + { + return list.Select(value => + { + if (Enum.TryParse(value, out T result) && Enum.IsDefined(result)) + { + return result; + } + throw new ArgumentException(value); + }).ToList(); + } + catch (ArgumentException e) + { + console.ErrorFmt( + "Failed to convert list of strings '{0}' to list of enums. Cause: {1}", + string.Join(", ", list), e.Message); + throw StarlarkRt.Errorf( + "Invalid value '{0}' for field '{1}'. Valid values are: {2}", + string.Join(", ", list), fieldName, string.Join(", ", Enum.GetNames())); + } + } + + /// Checks that a mandatory string field is not empty. + /// if the value is null or empty + public static string CheckNotEmpty(string? value, string name) + { + Check(!string.IsNullOrEmpty(value), "Invalid empty field '{0}'.", name); + return value!; + } + + /// Checks a condition or throws . + /// if the condition is false + public static void Check(bool condition, string format, params object?[] args) + { + if (!condition) + { + throw StarlarkRt.Errorf(format, args); + } + } + + /// + /// Converts a Starlark sequence value (such as a list or tuple) to a list of strings. The result + /// is a new, mutable copy. It throws EvalException if x is not a Starlark iterable or if any of + /// its elements are not strings. The message argument is prefixed to any error message. + /// + /// if x is not a sequence or an element is not a string + public static List ConvertStringList(object? x, string message) + { + if (x is not ISequence seq) + { + throw StarlarkRt.Errorf("{0}: got {1}, want sequence", message, StarlarkRt.Type(x)); + } + + var result = new List(); + foreach (object? elem in seq) + { + if (elem is not string s) + { + throw StarlarkRt.Errorf( + "{0}: at index #{1}, got {2}, want string", + message, result.Count, StarlarkRt.Type(elem)); + } + result.Add(s); + } + return result; + } + + /// + /// Converts a Starlark dict value to a map of strings to strings. The result is a new, mutable + /// copy. It throws EvalException if x is not a Starlark dict or if any of its keys or values are + /// not strings. The message argument is prefixed to any error message. + /// + /// if x is not a dict or a key/value is not a string + public static Dictionary ConvertStringMap(object? x, string message) + { + if (x is not Dict dict) + { + throw StarlarkRt.Errorf("{0}: got {1}, want dict", message, StarlarkRt.Type(x)); + } + var result = new Dictionary(); + foreach (var e in dict.Entries) + { + if (e.Key is not string key) + { + throw StarlarkRt.Errorf( + "{0}: in dict key, got {1}, want string", message, StarlarkRt.Type(e.Key)); + } + if (e.Value is not string value) + { + throw StarlarkRt.Errorf( + "{0}: in value for dict key '{1}', got {2}, want string", + message, e.Key, StarlarkRt.Type(e.Value)); + } + result[key] = value; + } + return result; + } + + /// + /// Converts a Starlark optional string value (string or None) to a nullable String reference. + /// + public static string? ConvertOptionalString(object? x) => + StarlarkRt.IsNullOrNone(x) ? null : (string)x!; +} diff --git a/src/Copybara.Core/Config/ValidationResult.cs b/src/Copybara.Core/Config/ValidationResult.cs new file mode 100644 index 000000000..9ffaf536e --- /dev/null +++ b/src/Copybara.Core/Config/ValidationResult.cs @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; + +namespace Copybara.Config; + +/// The immutable result of a validation performed by . +public sealed class ValidationResult +{ + public static readonly ValidationResult EMPTY = + new(ImmutableArray.Empty); + + private readonly ImmutableArray _messages; + + private ValidationResult(ImmutableArray messages) => _messages = messages; + + /// Returns all the s in the order they were registered. + public IReadOnlyList GetAllMessages() => _messages; + + /// Returns true iff there was at least one warning message. + public bool HasWarnings() => GetWarnings().Count > 0; + + /// Returns true iff there was at least one error message. + public bool HasErrors() => GetErrors().Count > 0; + + /// Returns the text of the warning messages, in the order that were registered. + public IReadOnlyList GetWarnings() => + _messages.Where(v => v.GetLevel() == Level.WARNING).Select(v => v.GetMessage()) + .ToImmutableArray(); + + /// Returns the text of the error messages, in the order that were registered. + public IReadOnlyList GetErrors() => + _messages.Where(v => v.GetLevel() == Level.ERROR).Select(v => v.GetMessage()) + .ToImmutableArray(); + + public override string ToString() => + $"ValidationResult{{messages=[{string.Join(", ", _messages)}]}}"; + + /// + /// Levels of validation messages. Can only be warning or error, because it doesn't make sense + /// to have info here. + /// + public enum Level + { + WARNING, + ERROR, + } + + /// Encapsulates a validation message and a . + public sealed class ValidationMessage + { + private readonly Level _level; + private readonly string _message; + + internal ValidationMessage(Level level, string message) + { + _level = level; + _message = Preconditions.CheckNotNull(message); + } + + public Level GetLevel() => _level; + + public string GetMessage() => _message; + + /// + /// Generates a string from this validation message with padded level and message text. + /// + public override string ToString() => string.Format("{0,-8} {1}", _level, _message); + } + + /// A builder of . + public sealed class Builder + { + private readonly List _messages = new(); + + public Builder Warning(string message) + { + _messages.Add(new ValidationMessage(Level.WARNING, message)); + return this; + } + + public Builder WarningFmt(string message, params object?[] args) + { + _messages.Add(new ValidationMessage(Level.WARNING, string.Format(message, args))); + return this; + } + + public Builder Error(string message) + { + _messages.Add(new ValidationMessage(Level.ERROR, message)); + return this; + } + + public Builder ErrorFmt(string message, params object?[] args) + { + _messages.Add(new ValidationMessage(Level.ERROR, string.Format(message, args))); + return this; + } + + public Builder Append(ValidationResult result) + { + _messages.AddRange(result._messages); + return this; + } + + public ValidationResult Build() => new(_messages.ToImmutableArray()); + } +} diff --git a/src/Copybara.Core/ConfigFileArgs.cs b/src/Copybara.Core/ConfigFileArgs.cs new file mode 100644 index 000000000..9925928a6 --- /dev/null +++ b/src/Copybara.Core/ConfigFileArgs.cs @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; + +namespace Copybara; + +/// +/// Arguments for a command that expects the CLI arguments be like: +/// config_file [workflow [source_ref]]. +/// +public sealed class ConfigFileArgs +{ + private readonly string _configPath; + private readonly string? _workflowName; + private readonly ImmutableArray _sourceRefs; + + public ConfigFileArgs(string configPath, string? workflowName) + : this(configPath, workflowName, ImmutableArray.Empty) + { + } + + public ConfigFileArgs(string configPath, string? workflowName, IEnumerable sourceRefs) + { + _configPath = Preconditions.CheckNotNull(configPath); + _workflowName = workflowName; + _sourceRefs = sourceRefs.ToImmutableArray(); + } + + public string GetConfigPath() => _configPath; + + public string GetWorkflowName() => _workflowName ?? "default"; + + public bool HasWorkflowName() => _workflowName != null; + + /// + /// Returns the first sourceRef from the command arguments, or null if no source ref was + /// provided. + /// + /// This method is provided for convenience, for subcommands that only care about the first + /// source_ref. + /// + public string? GetSourceRef() => _sourceRefs.Length == 0 ? null : _sourceRefs[0]; + + public IReadOnlyList GetSourceRefs() => _sourceRefs; +} diff --git a/src/Copybara.Core/ConfigGen/ConfigGenHeuristics.cs b/src/Copybara.Core/ConfigGen/ConfigGenHeuristics.cs new file mode 100644 index 000000000..929a2d4d1 --- /dev/null +++ b/src/Copybara.Core/ConfigGen/ConfigGenHeuristics.cs @@ -0,0 +1,801 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using Copybara; +using Copybara.Common; +using Copybara.Util; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.ConfigGen; + +/// +/// Given a set of files from the origin and a set of files from the destination, it generates +/// origin_globs, destination_globs and core.moves to minimize the number of transformations for +/// converting code from origin to destination. +/// +/// Note that the generation is not perfect and should be reviewed by a human. +/// +public class ConfigGenHeuristics +{ + // Regex to parse upstream semver-like version refs. Group 3 is expected to capture the separator + // character. + private static readonly Regex UpstreamVersionRefRegex = + new(@"^v?(\d+)(([^\d\n])(?:\d+)){0,2}$"); + + private readonly string _origin; + private readonly string _destination; + private readonly ImmutableHashSet _destinationOnlyPaths; + private readonly int _percentSimilar; + private readonly bool _ignoreCarriageReturn; + private readonly bool _ignoreWhitespace; + private readonly GeneralOptions _generalOptions; + private readonly ImmutableArray _versions; + + /// Creates the Generator object. + /// the root folder for the files of the origin repository. + /// the root folder for the files of the repository. + /// paths known to be only in the destination so they are + /// skipped in the similarity check. + /// percentage of similar lines to consider two files the same. + /// whether to ignore carriage return characters in file + /// content comparisons. + /// whether to ignore whitespace characters in file content + /// comparisons. + /// the object. + /// the list of version refs from the upstream. + public ConfigGenHeuristics( + string origin, + string destination, + ImmutableHashSet destinationOnlyPaths, + int percentSimilar, + bool ignoreCarriageReturn, + bool ignoreWhitespace, + GeneralOptions generalOptions, + ImmutableArray versions) + { + _origin = Preconditions.CheckNotNull(origin); + _destination = Preconditions.CheckNotNull(destination); + _destinationOnlyPaths = Preconditions.CheckNotNull(destinationOnlyPaths); + _percentSimilar = percentSimilar; + _ignoreCarriageReturn = ignoreCarriageReturn; + _ignoreWhitespace = ignoreWhitespace; + _generalOptions = generalOptions; + _versions = versions; + } + + /// Result of the config generation. + public sealed class Result + { + private readonly Glob _originGlob; + private readonly GeneratorTransformations _transformations; + private readonly DestinationExcludePaths _destinationExcludePaths; + private readonly bool _shouldUseVersionSelector; + private readonly string? _versionSeparator; + + public Result( + Glob originFiles, + GeneratorTransformations transformations, + DestinationExcludePaths destinationExcludePaths, + bool shouldUseVersionSelector, + string? versionSeparator) + { + _originGlob = originFiles; + _transformations = transformations; + _destinationExcludePaths = destinationExcludePaths; + _shouldUseVersionSelector = shouldUseVersionSelector; + _versionSeparator = versionSeparator; + } + + public Glob GetOriginGlob() => _originGlob; + + public GeneratorTransformations GetTransformations() => _transformations; + + public DestinationExcludePaths GetDestinationExcludePaths() => _destinationExcludePaths; + + public bool GetShouldUseVersionSelector() => _shouldUseVersionSelector; + + public string? GetVersionSeparator() => _versionSeparator; + } + + /// A path and its similarity score to some other path. + public readonly record struct PathAndScore(string Path, int Score); + + /// + /// Run the config generation to find a good origin_files, destination_files and core.moves needed + /// to convert the code from origin to destination. + /// + /// an object containing all the heuristic results. + public Result Run() + { + var gitFiles = ListFiles(_origin); + var g3Files = ListFiles(_destination); + var destinationToOriginMapping = + GetDestinationToOriginMapping(gitFiles, g3Files, _generalOptions.GetConsole()); + + // Map of Origin file paths to destination file paths. + // If multiple destination files map to the same origin file, we preserve the mapping with + // the highest score. + var similarFiles = new Dictionary(); + foreach (var group in destinationToOriginMapping.GroupBy(e => e.Value.Path)) + { + var best = group.Aggregate((a, b) => a.Value.Score >= b.Value.Score ? a : b); + similarFiles[best.Key] = group.Key; + } + + var originGlob = GetOriginGlob(gitFiles, similarFiles, g3Files); + var moves = GenerateMoves(similarFiles); + var destinationExcludePaths = + new DestinationExcludePaths( + GetDestinationExcludePaths(g3Files, similarFiles, _destinationOnlyPaths)); + string? tagSeparator = GetVersionStringSeparator(_versions); + + return new Result( + originGlob.GlobValue, + new GeneratorTransformations(moves), + destinationExcludePaths, + tagSeparator != null, + tagSeparator); + } + + /// Generates a mapping of destination files to origin files. + /// the list of paths in the origin. + /// the list of paths in the destination. + /// a map of destination to origin files, with their similarity scores. + protected Dictionary GetDestinationToOriginMapping( + ImmutableHashSet gitFiles, ImmutableHashSet g3Files, Console console) + { + var similarityDetector = + SimilarityDetector.Create( + _origin, + gitFiles, + _destinationOnlyPaths, + _percentSimilar, + _ignoreCarriageReturn, + _ignoreWhitespace); + // Map of destination file paths to origin file paths with similarity score. Sorted for + // deterministic behavior (Java uses a TreeMap). + var destinationToOriginMapping = + new SortedDictionary(StringComparer.Ordinal); + + foreach (var file in g3Files) + { + var originPathAndScore = similarityDetector.Find(PathOps.Resolve(_destination, file)); + // If we find an origin file with a higher similarity score to the destination file, map + // that origin file instead to the destination file. + if (originPathAndScore.HasValue) + { + var pathAndScore = originPathAndScore.Value; + if (!destinationToOriginMapping.TryGetValue(file, out var existing) + || pathAndScore.Score > existing.Score) + { + destinationToOriginMapping[file] = pathAndScore; + } + } + } + return new Dictionary(destinationToOriginMapping); + } + + private static string? GetVersionStringSeparator(ImmutableArray versions) + { + var separators = new Dictionary(); + if (versions.IsEmpty) + { + return null; + } + + foreach (var version in versions) + { + var matcher = UpstreamVersionRefRegex.Match(version); + if (matcher.Success && matcher.Groups.Count >= 4) + { + string sep = matcher.Groups[3].Value; + separators[sep] = separators.GetValueOrDefault(sep, 0) + 1; + } + } + + if (separators.Count == 0) + { + return null; + } + return separators.OrderByDescending(e => e.Value).First().Key; + } + + private IncludesGlob GetOriginGlob( + ImmutableHashSet gitFiles, + Dictionary similarFiles, + ImmutableHashSet g3Files) + { + var originOnly = new HashSet(gitFiles); + originOnly.ExceptWith(similarFiles.Keys); + + var destinationOnly = new HashSet(g3Files); + destinationOnly.ExceptWith(similarFiles.Values); + + var originGlob = + new IncludesGlob(ImmutableHashSet.Create("**"), ImmutableHashSet.Empty) + .MinimizeScore(similarFiles.Keys.ToList(), originOnly, 0); + + // Enable to debug what is being generated: + if (_generalOptions.IsVerbose()) + { + Debug(similarFiles, destinationOnly, originGlob); + } + + return ConsolidateCommonPattern( + originGlob, similarFiles.Keys.ToHashSet(), p => p.StartsWith('.'), ".**"); + } + + /// + /// Returns the set of files that are in the destination but not in the origin. This is the union + /// of the known destinationOnlyPaths and the files that are in g3Files but not in similarFiles. + /// + private ImmutableHashSet GetDestinationExcludePaths( + ImmutableHashSet g3Files, + Dictionary similarFiles, + ImmutableHashSet destinationOnlyPaths) + { + var similarValues = similarFiles.Values.ToHashSet(); + return destinationOnlyPaths + .Union(g3Files.Where(p => !similarValues.Contains(p))) + .ToImmutableHashSet(); + } + + /// + /// Generates the minimal amount of core.moves to map files from the origin to the destination. + /// See the Java documentation for the full algorithm description. + /// + private ImmutableArray GenerateMoves(Dictionary similarFiles) + { + var set = new LinkedList>(similarFiles); + var result = new MovesTrie(); + while (set.Count > 0) + { + var entry = set.First!.Value; + set.RemoveFirst(); + string origin = entry.Key; + string dest = entry.Value; + if (origin == dest) + { + // already correctly positioned + continue; + } + string commonSuffix = CommonSuffix(origin, dest); + bool handled = false; + while (commonSuffix.Length != 0) + { + int suffixCount = NameCount(commonSuffix); + string originPrefix = + suffixCount != NameCount(origin) + ? Subpath(origin, 0, NameCount(origin) - suffixCount) + : ""; + string destPrefix = + suffixCount != NameCount(dest) + ? Subpath(dest, 0, NameCount(dest) - suffixCount) + : ""; + bool tooBroad = false; + var includedPaths = new HashSet>(); + foreach (var e in similarFiles) + { + if (StartsWith(e.Key, originPrefix)) + { + string relocated = + Resolve( + destPrefix, + Subpath(e.Key, NameCount(originPrefix), NameCount(e.Key))); + if (relocated == e.Value) + { + includedPaths.Add(e); + } + else + { + tooBroad = true; + break; + } + } + } + if (tooBroad) + { + commonSuffix = suffixCount == 1 + ? "" // 'foo' -> "", subpath doesn't work here. + : Subpath(commonSuffix, 1, suffixCount); + } + else + { + // Successfully moves a bunch of files with a directory move. + foreach (var p in includedPaths) + { + set.Remove(p); + } + result.InsertMove(new GeneratorMove(originPrefix, destPrefix)); + handled = true; + break; + } + } + if (!handled) + { + result.InsertMove(new GeneratorMove(origin, dest)); + } + } + return result.GetMovesInOrder(); + } + + private void Debug( + Dictionary similarFiles, + HashSet destinationOnly, + IncludesGlob originGlob) + { + Console console = _generalOptions.GetConsole(); + foreach (var e in similarFiles) + { + console.Verbose(e.Key + " -> " + e.Value); + } + + console.Verbose("git_files = " + originGlob.GlobValue); + + foreach (var path in destinationOnly) + { + console.Verbose("G3 Only: " + path); + } + } + + /// + /// Consolidate more than one pattern that matches the predicate with a single replacement pattern + /// when there is no file being migrated that matches the predicate. + /// + private IncludesGlob ConsolidateCommonPattern( + IncludesGlob originGlob, + HashSet migratedFiles, + Func filePredicate, + string replacement) + { + if (originGlob.Excludes.Count(filePredicate) <= 1 + || migratedFiles.Any(p => filePredicate(p))) + { + return originGlob; + } + var newExcludes = ImmutableHashSet.CreateBuilder(); + newExcludes.Add(replacement); + foreach (var pattern in originGlob.Excludes) + { + if (filePredicate(pattern)) + { + continue; + } + newExcludes.Add(pattern); + } + return new IncludesGlob(originGlob.Includes, newExcludes.ToImmutable()); + } + + private sealed class ExcludesGlob : IncludesGlob + { + internal ExcludesGlob(IReadOnlySet includes, IReadOnlySet excludes) + : base(includes, excludes) + { + } + + protected override int Score() => + // If excludes needs excludes, it is not a good excludes. + Excludes.Count == 0 ? base.Score() : int.MaxValue; + + protected override IncludesGlob WithExcludes( + IReadOnlyCollection toBeIncluded, IReadOnlySet toBeExcluded) + { + var matchingExcludes = FindMatchingExcludes(toBeExcluded); + return Create(Includes, matchingExcludes); + } + + protected override IncludesGlob Create(IReadOnlySet includes, IReadOnlySet excludes) => + new ExcludesGlob(includes, excludes); + } + + private class IncludesGlob : IComparable + { + internal readonly IReadOnlySet Includes; + internal readonly IReadOnlySet Excludes; + internal readonly Glob GlobValue; + + internal IncludesGlob(IReadOnlySet includes, IReadOnlySet excludes) + { + Includes = includes; + Excludes = excludes; + GlobValue = Glob.CreateGlob(includes, excludes); + } + + protected virtual IncludesGlob Create(IReadOnlySet includes, IReadOnlySet excludes) => + // Use sorted set to have it sorted. + new IncludesGlob( + new SortedSet(includes, StringComparer.Ordinal), + new SortedSet(excludes, StringComparer.Ordinal)); + + protected virtual int Score() => + Math.Max(Includes.Count, 1) * Math.Max(Excludes.Count, 1); + + public int CompareTo(IncludesGlob? o) => + o == null ? 1 : Score().CompareTo(o.Score()); + + internal IncludesGlob MinimizeScore( + IReadOnlyCollection toBeIncluded, IReadOnlySet toBeExcluded, int level) + { + IncludesGlob globAndScore = WithExcludes(toBeIncluded, toBeExcluded); + + var recursiveIncludes = new Dictionary>(); + var newIncludes = new HashSet(); + var newExcludes = new HashSet(); + foreach (var p in toBeIncluded) + { + if (NameCount(p) <= level + 1) + { + newIncludes.Add(p); + } + else + { + string key = Subpath(p, 0, level + 1) + "/**"; + if (!recursiveIncludes.TryGetValue(key, out var bucket)) + { + bucket = new HashSet(); + recursiveIncludes[key] = bucket; + } + bucket.Add(p); + } + } + // For each recursive pattern, try to optimize for fewer entries and see if the + // combination has fewer entries than globAndScore. + foreach (var pattern in recursiveIncludes.Keys) + { + IncludesGlob newGlob = + Create(ImmutableHashSet.Create(pattern), ImmutableHashSet.Empty) + .MinimizeScore(recursiveIncludes[pattern].ToList(), toBeExcluded, level + 1); + newIncludes.UnionWith(newGlob.Includes); + newExcludes.UnionWith(newGlob.Excludes); + } + IncludesGlob comboGlob = Create(newIncludes, newExcludes); + + return comboGlob.Score() < globAndScore.Score() ? comboGlob : globAndScore; + } + + protected virtual IncludesGlob WithExcludes( + IReadOnlyCollection toBeIncluded, IReadOnlySet toBeExcluded) + { + var excludedPaths = FindMatchingExcludes(toBeExcluded); + IncludesGlob optimizedExcludes = + new ExcludesGlob(ImmutableHashSet.Create("**"), ImmutableHashSet.Empty) + .MinimizeScore(excludedPaths.ToList(), toBeIncluded.ToImmutableHashSet(), 0); + + // Found a better excludes than the naive approach of listing all excludes! + if (optimizedExcludes.Excludes.Count == 0) + { + return Create(Includes, optimizedExcludes.Includes); + } + + return Create(Includes, excludedPaths); + } + + protected IReadOnlySet FindMatchingExcludes(IReadOnlySet toBeExcluded) + { + IPathMatcher pathMatcher = Glob.CreateGlob(Includes).RelativeTo("/"); + var excludedPaths = ImmutableHashSet.CreateBuilder(); + foreach (var ex in toBeExcluded) + { + if (pathMatcher.Matches(PathOps.Resolve("/", ex))) + { + excludedPaths.Add(ex); + } + } + return excludedPaths.ToImmutable(); + } + + public override string ToString() => + $"{GetType().Name}(score: {Score()}, {GlobValue})"; + } + + private static string CommonSuffix(string a, string b) + { + var aNames = Names(a); + var bNames = Names(b); + var paths = new LinkedList(); + for (int aIndex = aNames.Length - 1, bIndex = bNames.Length - 1; + aIndex >= 0 && bIndex >= 0; + aIndex--, bIndex--) + { + if (aNames[aIndex] != bNames[bIndex]) + { + break; + } + paths.AddFirst(aNames[aIndex]); + } + return string.Join("/", paths); + } + + /// + /// Detects similar files based on hash and similarity. Used to find similar files in the origin + /// and destination, and map them together. + /// + protected sealed class SimilarityDetector + { + // Useful for binaries. + private readonly ImmutableListMultimap _hashBased; + private readonly RenameDetector _similarLines; + private readonly ImmutableHashSet _destinationOnlyPaths; + private readonly int _percentSimilar; + + private SimilarityDetector( + ImmutableListMultimap hashBased, + RenameDetector similarLines, + ImmutableHashSet destinationOnlyPaths, + int percentSimilar) + { + _hashBased = hashBased; + _similarLines = similarLines; + _destinationOnlyPaths = destinationOnlyPaths; + _percentSimilar = percentSimilar; + } + + internal PathAndScore? Find(string path) + { + if (_destinationOnlyPaths.Contains(PathOps.GetFileName(path))) + { + return null; + } + + byte[] content = File.ReadAllBytes(path); + + // Highest priority same hash. RenameDetector fails for small files. + string? hashFinding = null; + int bestSuffix = -1; + foreach (var o in _hashBased.Get(Hash(content))) + { + int suffix = NameCount(CommonSuffix(path, o)); + if (suffix > bestSuffix) + { + bestSuffix = suffix; + hashFinding = o; + } + } + if (hashFinding != null) + { + return new PathAndScore(hashFinding, RenameDetector.MaxScore); + } + + // Second priority similarity. + var scores = _similarLines.ScoresForLaterFile(new MemoryStream(content)); + var score = scores.Length > 0 ? scores[0] : null; + if (score != null + && score.GetScore() > RenameDetector.MaxScore * _percentSimilar / 100) + { + return new PathAndScore(score.GetKey(), score.GetScore()); + } + return null; + } + + internal static SimilarityDetector Create( + string parent, + ImmutableHashSet files, + ImmutableHashSet destinationOnlyPaths, + int percentSimilar, + bool ignoreCarriageReturn, + bool ignoreWhitespace) + { + var similarLines = + new RenameDetector( + ignoreCarriageReturn, ignoreWhitespace, skipNewlinesInHash: true); + var hashes = ImmutableListMultimap.CreateBuilder(); + foreach (var file in files) + { + byte[] bytes = File.ReadAllBytes(PathOps.Resolve(parent, file)); + hashes.Put(Hash(bytes), file); + similarLines.AddPriorFile(file, new MemoryStream(bytes)); + } + return new SimilarityDetector( + hashes.Build(), similarLines, destinationOnlyPaths, percentSimilar); + } + + private static string Hash(byte[] bytes) => + Convert.ToHexStringLower(SHA256.HashData(bytes)); + } + + private static ImmutableHashSet ListFiles(string path) + { + var result = ImmutableHashSet.CreateBuilder(); + if (!Directory.Exists(path)) + { + return result.ToImmutable(); + } + foreach (var file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)) + { + var attrs = File.GetAttributes(file); + if ((attrs & FileAttributes.ReparsePoint) != 0) + { + continue; + } + result.Add(PathOps.Relativize(path, file)); + } + return result.ToImmutable(); + } + + /// Represents a core.move() to be included in the generation. + public sealed class GeneratorMove : IEquatable + { + private readonly string _before; + private readonly string _after; + + public GeneratorMove(string before, string after) + { + _before = before; + _after = after; + } + + public string GetBefore() => _before; + + public string GetAfter() => _after; + + public bool Equals(GeneratorMove? that) => + that is not null && _before == that._before && _after == that._after; + + public override bool Equals(object? o) => Equals(o as GeneratorMove); + + public override int GetHashCode() => HashCode.Combine(_before, _after); + + public override string ToString() => $"core.move(\"{_before}\", \"{_after}\")"; + } + + /// Represents a collection of transformations to be included in the generation. + public sealed class GeneratorTransformations + { + private readonly ImmutableArray _moves; + + public GeneratorTransformations(ImmutableArray moves) + { + _moves = moves; + } + + public ImmutableArray GetMoves() => _moves; + } + + /// + /// Represents a collection of paths that are found to only be present in the destination. This + /// should be a union of the given destinationOnlyPaths and files found to only exist in the + /// destination. + /// + public sealed class DestinationExcludePaths + { + private readonly ImmutableHashSet _paths; + + public DestinationExcludePaths(ImmutableHashSet paths) + { + _paths = paths; + } + + public ImmutableHashSet GetPaths() => _paths; + + public override string ToString() => string.Join(", ", _paths); + } + + /// + /// A prefix tree (trie) used to maintain the order of core.move transforms, such that each + /// transform does not interfere with the operations of another. + /// + private sealed class MovesTrie + { + private readonly MovesTrieNode _root = new(); + + private sealed class MovesTrieNode + { + private readonly Dictionary _children = new(); + private GeneratorMove? _move; + + public GeneratorMove? GetMove() => _move; + + public void SetMove(GeneratorMove move) => _move = move; + + public void AddChildren(string path, MovesTrieNode node) => _children[path] = node; + + public IReadOnlyDictionary GetChildren() => _children; + } + + public void InsertMove(GeneratorMove move) + { + MovesTrieNode currentNode = _root; + + // Edge case - if the before path is an empty path, this means it's moving the root. + if (move.GetBefore().Length == 0) + { + currentNode.SetMove(move); + return; + } + + foreach (var name in Names(move.GetBefore())) + { + if (!currentNode.GetChildren().ContainsKey(name)) + { + currentNode.AddChildren(name, new MovesTrieNode()); + } + currentNode = currentNode.GetChildren()[name]; + } + currentNode.SetMove(move); + } + + public ImmutableArray GetMovesInOrder() => GetMovesInOrder(_root); + + private static ImmutableArray GetMovesInOrder(MovesTrieNode startNode) + { + var moves = ImmutableArray.CreateBuilder(); + foreach (var child in startNode.GetChildren().Values) + { + moves.AddRange(GetMovesInOrder(child)); + } + var move = startNode.GetMove(); + if (move != null) + { + moves.Add(move); + } + return moves.ToImmutable(); + } + } + + // ---- java.nio.file.Path shims (forward-slash separated string paths) ---- + + private static string[] Names(string path) => + path.Split('/', StringSplitOptions.RemoveEmptyEntries); + + private static int NameCount(string path) => Names(path).Length; + + /// Equivalent of Path.subpath(begin, end). + private static string Subpath(string path, int begin, int end) + { + var names = Names(path); + return string.Join("/", names[begin..end]); + } + + /// Equivalent of Path.startsWith(other) on name components. + private static bool StartsWith(string path, string prefix) + { + if (prefix.Length == 0) + { + return true; + } + var pathNames = Names(path); + var prefixNames = Names(prefix); + if (prefixNames.Length > pathNames.Length) + { + return false; + } + for (int i = 0; i < prefixNames.Length; i++) + { + if (pathNames[i] != prefixNames[i]) + { + return false; + } + } + return true; + } + + /// Equivalent of base.resolve(other). + private static string Resolve(string basePath, string other) + { + if (other.Length == 0) + { + return basePath; + } + if (basePath.Length == 0) + { + return other; + } + return basePath + "/" + other; + } +} diff --git a/src/Copybara.Core/ConfigItemDescription.cs b/src/Copybara.Core/ConfigItemDescription.cs new file mode 100644 index 000000000..9dc90b098 --- /dev/null +++ b/src/Copybara.Core/ConfigItemDescription.cs @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Util; + +namespace Copybara; + +/// +/// Interface for self-description. The information returned should be sufficient to create a new +/// instance with identical migration behavior (but potentially different side effects). This is +/// intended for discovering changes in a config. +/// +public interface IConfigItemDescription +{ + string GetTypeName() => GetType().FullName ?? GetType().Name; + + /// Returns a key-value list of the options the endpoint was instantiated with. + ImmutableListMultimap Describe(Glob? originFiles) + { + var builder = ImmutableListMultimap.CreateBuilder(); + builder.Put("type", GetTypeName()); + return builder.Build(); + } + + /// Returns a key-value list describing the credentials the endpoint was instantiated with. + IReadOnlyList> DescribeCredentials() => + ImmutableArray>.Empty; + + /// Returns a key-value list describing the credentials the endpoint was instantiated with. + IReadOnlyList> DescribeCredentials(string endpoint) + { + var creds = DescribeCredentials(); + if (creds.Count == 0) + { + return creds; + } + var builder = ImmutableArray.CreateBuilder>(); + foreach (var cred in creds) + { + var credBuilder = ImmutableListMultimap.CreateBuilder(); + credBuilder.PutAll(cred); + credBuilder.Put("endpoint", endpoint); + builder.Add(credBuilder.Build()); + } + return builder.ToImmutable(); + } +} diff --git a/src/Copybara.Core/ConfigLoader.cs b/src/Copybara.Core/ConfigLoader.cs new file mode 100644 index 000000000..2131f9df6 --- /dev/null +++ b/src/Copybara.Core/ConfigLoader.cs @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Copybara.Config; +using Copybara.Exceptions; +using Copybara.Revision; +using Copybara.Util.Console; +using Console = Copybara.Util.Console.Console; + +namespace Copybara; + +/// Loads the configuration from a given config file. +public class ConfigLoader +{ + private readonly SkylarkParser _skylarkParser; + protected readonly ConfigFile ConfigFile; + private readonly ModuleSet _moduleSet; + + public ConfigLoader(ModuleSet moduleSet, ConfigFile configFile, StarlarkMode validateStarlark) + { + _moduleSet = moduleSet; + _skylarkParser = new SkylarkParser(_moduleSet.GetStaticModules(), validateStarlark); + ConfigFile = Preconditions.CheckNotNull(configFile); + } + + /// Returns a string representation of the location of this configuration. + public string Location() => ConfigFile.Path(); + + /// Loads the configuration using this loader. + /// the console to use for reporting progress/errors. + public Config.Config Load(Console console) => LoadForConfigFile(console, ConfigFile); + + /// Loads the configuration and its dependencies using this loader. + /// the console to use for reporting progress/errors. + public ConfigWithDependencies LoadWithDependencies(Console console) + { + console.ProgressFmt("Loading config and dependencies {0}", ConfigFile.GetIdentifier()); + + using (_moduleSet.GetOptions().Get().Profiler() + .Start("loading_config_with_deps")) + { + return _skylarkParser.GetConfigWithTransitiveImports(ConfigFile, _moduleSet, console); + } + } + + protected Config.Config LoadForConfigFile(Console console, ConfigFile configFile) + { + console.ProgressFmt("Loading config {0}", configFile.GetIdentifier()); + + using (_moduleSet.GetOptions().Get().Profiler().Start("loading_config")) + { + return _skylarkParser.LoadConfig(configFile, _moduleSet, console); + } + } + + protected virtual Config.Config DoLoadForRevision(Console console, IRevision revision) => + throw new NotSupportedException( + "This origin/configuration doesn't allow loading configs from specific revisions"); + + public Config.Config LoadForRevision(Console console, IRevision revision) + { + using (_moduleSet.GetOptions().Get().Profiler() + .Start("loading_config_for_revision")) + { + return DoLoadForRevision(console, revision); + } + } + + public virtual bool SupportsLoadForRevision() => false; +} diff --git a/src/Copybara.Core/ConsistencyFileConfiguration.cs b/src/Copybara.Core/ConsistencyFileConfiguration.cs new file mode 100644 index 000000000..362510116 --- /dev/null +++ b/src/Copybara.Core/ConsistencyFileConfiguration.cs @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2026 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara; + +/// An object used to configure Consistency File options. +[StarlarkBuiltin("core.consistency_file_config", Documented = true)] +public sealed class ConsistencyFileConfiguration : IStarlarkValue +{ + private readonly string _path; + private readonly bool _excludeBuildFiles; + + private ConsistencyFileConfiguration(string path, bool excludeBuildFiles) + { + _path = path; + _excludeBuildFiles = excludeBuildFiles; + } + + public static ConsistencyFileConfiguration Create(string path, bool excludeBuildFiles) => + new(path, excludeBuildFiles); + + public string Path() => _path; + + public bool ExcludeBuildFiles() => _excludeBuildFiles; +} diff --git a/src/Copybara.Core/ContextProvider.cs b/src/Copybara.Core/ContextProvider.cs new file mode 100644 index 000000000..dac6824f7 --- /dev/null +++ b/src/Copybara.Core/ContextProvider.cs @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Config; +using Console = Copybara.Util.Console.Console; + +namespace Copybara; + +/// A class providing additional context for CMD. +public interface IContextProvider +{ + /// Get context for CMD. + IReadOnlyDictionary GetContext( + Config.Config config, + ConfigFileArgs configFileArgs, + IConfigLoaderProvider configLoaderProvider, + Console console); + + /// Get context for CMD. + IReadOnlyDictionary GetContext( + ConfigWithDependencies config, + ConfigFileArgs configFileArgs, + IConfigLoaderProvider configLoaderProvider, + Options options, + Console console) => + GetContext(config.GetConfig(), configFileArgs, configLoaderProvider, console); +} diff --git a/src/Copybara.Core/ConvertEncoding.cs b/src/Copybara.Core/ConvertEncoding.cs new file mode 100644 index 000000000..526a4cf63 --- /dev/null +++ b/src/Copybara.Core/ConvertEncoding.cs @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text; +using Copybara.TreeState; +using Copybara.Util; + +namespace Copybara; + +/// Convert encoding for a set of files. +public class ConvertEncoding : ITransformation +{ + private readonly Encoding _before; + private readonly Encoding _after; + private readonly Glob _paths; + + public ConvertEncoding(Encoding before, Encoding after, Glob paths) + { + _before = before; + _after = after; + _paths = paths; + } + + public TransformationStatus Transform(TransformWork work) + { + string checkoutDir = work.GetCheckoutDir(); + var files = new HashSet(); + foreach (var f in work.GetTreeState().Find(_paths.RelativeTo(checkoutDir))) + { + byte[] raw = File.ReadAllBytes(f.GetPath()); + string content = _before.GetString(raw); + File.WriteAllBytes(f.GetPath(), _after.GetBytes(content)); + files.Add(f); + } + work.GetTreeState().NotifyModify(files); + return files.Count == 0 + ? TransformationStatus.Noop("Glob didn't match any file") + : TransformationStatus.Success(); + } + + public ITransformation Reverse() => new ConvertEncoding(_after, _before, _paths); + + public string Describe() => "convert_encoding"; +} diff --git a/src/Copybara.Core/Copybara.Core.csproj b/src/Copybara.Core/Copybara.Core.csproj new file mode 100644 index 000000000..47a90ee1f --- /dev/null +++ b/src/Copybara.Core/Copybara.Core.csproj @@ -0,0 +1,15 @@ + + + Copybara + Copybara.Core + + + + + + + + + + + diff --git a/src/Copybara.Core/CoreGlobal.cs b/src/Copybara.Core/CoreGlobal.cs new file mode 100644 index 000000000..2090d476e --- /dev/null +++ b/src/Copybara.Core/CoreGlobal.cs @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Authoring; +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; + +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara; + +/// +/// A module to expose Starlark glob(), parse_message(), etc. functions. +/// +/// Don't add functions here and prefer the "core" namespace unless it is something really +/// general. +/// +public sealed class CoreGlobal : IStarlarkValue +{ + [StarlarkMethod("glob", + Doc = + "Returns an object which matches every file in the workdir that matches at least one" + + " pattern in include and does not match any of the patterns in exclude.")] + public Glob Glob( + [Param(Name = "include", Named = true, + Doc = "The list of glob patterns to include", + AllowedTypes = new[] { typeof(StarlarkList) })] + StarlarkList include, + [Param(Name = "exclude", Named = true, Positional = false, DefaultValue = "[]", + Doc = "The list of glob patterns to exclude", + AllowedTypes = new[] { typeof(StarlarkList) })] + StarlarkList exclude) + { + var includeStrings = ConvertStringList(include, "include"); + var excludeStrings = ConvertStringList(exclude, "exclude"); + try + { + return Copybara.Util.Glob.CreateGlob(includeStrings, excludeStrings); + } + catch (ArgumentException e) + { + throw StarlarkRt.Errorf( + "Cannot create a glob from: include='{0}' and exclude='{1}': {2}", + string.Join(", ", includeStrings), string.Join(", ", excludeStrings), e.Message); + } + } + + [StarlarkMethod("parse_message", + Doc = "Returns a ChangeMessage parsed from a well formed string.")] + public ChangeMessage ParseMessage( + [Param(Name = "message", Named = true, Doc = "The contents of the change message")] + string changeMessage) + { + try + { + return ChangeMessage.ParseMessage(changeMessage); + } + catch (Exception e) + { + throw StarlarkRt.Errorf( + "Cannot parse change message '{0}': {1}", changeMessage, e.Message); + } + } + + [StarlarkMethod("new_author", + Doc = "Create a new author from a string with the form 'name '")] + public Author NewAuthor( + [Param(Name = "author_string", Named = true, + Doc = "A string representation of the author with the form 'name '")] + string authorString) => + Author.Parse(authorString); + + // Port of SkylarkUtil.convertStringList (that helper is being ported concurrently in + // Copybara.Config). Inlined here to keep CoreGlobal self-contained. + internal static List ConvertStringList(IEnumerable list, string name) + { + var result = new List(); + foreach (var o in list) + { + if (o is not string s) + { + throw StarlarkRt.Errorf( + "Expected a string for element of '{0}', but got {1}", name, StarlarkRt.Type(o)); + } + + result.Add(s); + } + + return result; + } +} diff --git a/src/Copybara.Core/CoreModule.cs b/src/Copybara.Core/CoreModule.cs new file mode 100644 index 000000000..c81c15bf1 --- /dev/null +++ b/src/Copybara.Core/CoreModule.cs @@ -0,0 +1,1493 @@ +/* + * Copyright (C) 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.RegularExpressions; +using Copybara.Authoring; +using Copybara.Common; +using Copybara.Config; +using Copybara.Exceptions; +using Copybara.TemplateToken; +using Copybara.Transform; +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; +using Starlark.Syntax; + +// Domain 'Console' collides with System.Console (not used here directly) and static 'Starlark' +// collides with the root namespace segment. +using StarlarkRt = Starlark.Eval.Starlark; + +// Java's net.starlark.java.eval.Sequence (the interface for Starlark lists/tuples) maps to the +// concrete StarlarkList in this port; the static helper class Starlark.Eval.Sequence is unrelated. +using StarlarkSequence = Starlark.Eval.StarlarkList; +using StructImpl = Copybara.StructModule.StructImpl; + +// 'Authoring' is both a namespace (Copybara.Authoring) and a type; alias the type to disambiguate. +using AuthoringType = Copybara.Authoring.Authoring; + +namespace Copybara; + +/// +/// Main configuration class for creating migrations. +/// +/// This class is exposed in Starlark configuration as an instance variable called "core". So +/// users can use it as: +/// +/// core.workflow( +/// name = "foo", +/// ... +/// ) +/// +/// +[StarlarkBuiltin( + "core", + Doc = "Core functionality for creating migrations, and basic transformations.")] +public class CoreModule : ILabelsAwareModule, IStarlarkValue +{ + // Restrict for label ids like 'BAZEL_REV_ID' or 'Bazel-RevId'. + private static readonly Regex CustomRevidFormat = + new("^([A-Z][A-Z_0-9]{1,30}_REV_ID|[A-Z][a-zA-Z0-9-]{1,30}-RevId)$"); + + private const string CheckLastRevStateName = "check_last_rev_state"; + + private readonly GeneralOptions _generalOptions; + private readonly WorkflowOptions _workflowOptions; + + // TODO(port): reconcile — DebugOptions and FolderModule are being ported concurrently. + private readonly Copybara.Transform.Debug.DebugOptions _debugOptions; + private readonly Copybara.Folder.FolderModule _folderModule; + + private ConfigFile _mainConfigFile = null!; + private Func>? _allConfigFiles; + private StarlarkThread.PrintHandler? _printHandler; + private readonly Dictionary> _transformNames = new(); + private SkylarkConsole? _console; + private readonly object _consoleLock = new(); + + public CoreModule( + GeneralOptions generalOptions, + WorkflowOptions workflowOptions, + Copybara.Transform.Debug.DebugOptions debugOptions, + Copybara.Folder.FolderModule folderModule) + { + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _workflowOptions = Preconditions.CheckNotNull(workflowOptions); + _debugOptions = Preconditions.CheckNotNull(debugOptions); + _folderModule = Preconditions.CheckNotNull(folderModule); + } + + [StarlarkMethod("reverse", + Doc = + "Given a list of transformations, returns the list of transformations equivalent to" + + " undoing all the transformations")] + public StarlarkSequence Reverse( + [Param(Name = "transformations", Named = true, + Doc = "The transformations to reverse", + AllowedTypes = new[] { typeof(StarlarkSequence) })] + StarlarkSequence transforms) + { + var builder = ImmutableArray.CreateBuilder(); + foreach (var t in transforms) + { + try + { + builder.Add(Transformations.ToTransformation(t, "transformations", _printHandler).Reverse()); + } + catch (NonReversibleValidationException e) + { + throw StarlarkRt.Errorf("{0} at {1}", e.Message, GetLocation(t)); + } + } + + var reversed = builder.ToImmutable(); + return StarlarkList.ImmutableCopyOf(reversed.Reverse().Cast()); + } + + private static Location GetLocation(object? transformationOrCallable) => + transformationOrCallable switch + { + IStarlarkCallable callable => callable.Location, + ITransformation transformation => transformation.Location(), + _ => Location.BUILTIN, + }; + + [StarlarkMethod("workflow", + Doc = "Defines a migration pipeline which can be invoked via the Copybara command.", + UseStarlarkThread = true)] + public void Workflow( + [Param(Name = "name", Named = true, Positional = false, Doc = "The name of the workflow.")] + string workflowName, + [Param(Name = "origin", Named = true, Positional = false, + Doc = "Where to read from the code to be migrated, before applying the transformations.")] + object origin, + [Param(Name = "destination", Named = true, Positional = false, + Doc = "Where to write to the code being migrated, after applying the transformations.")] + object destination, + [Param(Name = "authoring", Named = true, Positional = false, + Doc = "The author mapping configuration from origin to destination.")] + AuthoringType authoring, + [Param(Name = "transformations", Named = true, Positional = false, DefaultValue = "[]", + Doc = "The transformations to be run for this workflow. They will run in sequence.")] + StarlarkSequence transformations, + [Param(Name = "origin_files", Named = true, Positional = false, DefaultValue = "None", + Doc = "A glob or list of files relative to the workdir that will be read from the origin.", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object originFiles, + [Param(Name = "destination_files", Named = true, Positional = false, DefaultValue = "None", + Doc = "A glob relative to the root of the destination repository.", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object destinationFiles, + [Param(Name = "mode", Named = true, Positional = false, DefaultValue = "\"SQUASH\"", + Doc = "Workflow mode: SQUASH, ITERATIVE, CHANGE_REQUEST, CHANGE_REQUEST_FROM_SOT.")] + string modeStr, + [Param(Name = "reversible_check", Named = true, Positional = false, DefaultValue = "None", + Doc = "Indicates if the tool should try to reverse all the transformations at the end.", + AllowedTypes = new[] { typeof(bool), typeof(NoneType) })] + object reversibleCheckObj, + [Param(Name = CheckLastRevStateName, Named = true, Positional = false, DefaultValue = "False", + Doc = "If set to true, Copybara will validate that the destination didn't change.")] + bool checkLastRevState, + [Param(Name = "ask_for_confirmation", Named = true, Positional = false, DefaultValue = "False", + Doc = "Show the diff and require the user's confirmation before making a change.")] + bool askForConfirmation, + [Param(Name = "dry_run", Named = true, Positional = false, DefaultValue = "False", + Doc = "Run the migration in dry-run mode.")] + bool dryRunMode, + [Param(Name = "after_migration", Named = true, Positional = false, DefaultValue = "[]", + Doc = "Run a feedback workflow after one migration happens.")] + StarlarkSequence afterMigrations, + [Param(Name = "after_workflow", Named = true, Positional = false, DefaultValue = "[]", + Doc = "Run a feedback workflow after all the changes for this workflow run are migrated.")] + StarlarkSequence afterAllMigrations, + [Param(Name = "change_identity", Named = true, Positional = false, DefaultValue = "None", + Doc = "Customize the identity hash generation.", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object changeIdentityObj, + [Param(Name = "set_rev_id", Named = true, Positional = false, DefaultValue = "True", + Doc = "Whether Copybara adds labels like 'GitOrigin-RevId' in the destination.")] + bool setRevId, + [Param(Name = "smart_prune", Named = true, Positional = false, DefaultValue = "False", + Doc = "Best-effort approach at restoring the non-affected snippets.")] + bool smartPrune, + [Param(Name = "merge_import", Named = true, Positional = false, DefaultValue = "None", + Doc = "A migration mode that shells out to a diffing tool to merge all files.", + AllowedTypes = new[] { typeof(bool), typeof(MergeImportConfiguration), typeof(NoneType) })] + object mergeImportObj, + [Param(Name = "autopatch_config", Named = true, Positional = false, DefaultValue = "None", + Doc = "Configuration that describes the setting for automatic patch file generation.", + AllowedTypes = new[] { typeof(AutoPatchfileConfiguration), typeof(NoneType) })] + object autoPatchFileConfigurationObj, + [Param(Name = "after_merge_transformations", Named = true, Positional = false, DefaultValue = "[]", + Doc = "Perform these transformations after merge_import.")] + StarlarkSequence afterMergeTransformations, + [Param(Name = "migrate_noop_changes", Named = true, Positional = false, DefaultValue = "False", + Doc = "Include all the changes, not only those affecting origin_files or config files.")] + bool migrateNoopChanges, + [Param(Name = "experimental_custom_rev_id", Named = true, Positional = false, DefaultValue = "None", + Doc = "DEPRECATED. Use custom_rev_id.", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object experimentalCustomRevIdField, + [Param(Name = "custom_rev_id", Named = true, Positional = false, DefaultValue = "None", + Doc = "Use this label name instead of the one provided by the origin.", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object customRevIdField, + [Param(Name = "description", Named = true, Positional = false, DefaultValue = "None", + Doc = "A description of what this workflow achieves", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object description, + [Param(Name = "checkout", Named = true, Positional = false, DefaultValue = "True", + Doc = "Allows disabling the checkout.")] + bool checkout, + [Param(Name = "reversible_check_ignore_files", Named = true, Positional = false, DefaultValue = "None", + Doc = "Ignore the files matching the glob in the reversible check", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object reversibleCheckIgnoreFiles, + [Param(Name = "consistency_file_path", Named = true, Positional = false, DefaultValue = "None", + Doc = "Under development. Must end with .bara.consistency", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object consistencyFilePathObj, + [Param(Name = "consistency_file", Named = true, Positional = false, DefaultValue = "None", + Doc = "Consistency file configuration. Can be a boolean or a consistency_file_config object.", + AllowedTypes = new[] { typeof(bool), typeof(ConsistencyFileConfiguration), typeof(NoneType) })] + object consistencyFileObj, + StarlarkThread thread) + { + var mode = SkylarkUtil.StringToEnum("mode", modeStr); + + // Overwrite destination for testing workflow locally. + if (_workflowOptions.ToFolder) + { + destination = _folderModule.Destination(); + } + + var sequenceTransform = + Copybara.Transform.Sequence.FromConfig( + _generalOptions.Profiler(), + null, + _workflowOptions, + transformations, + "transformations", + _printHandler, + _debugOptions.TransformWrapper, + Copybara.Transform.Sequence.NoopBehavior.NOOP_IF_ANY_NOOP); + + ITransformation? reverseTransform = null; + if (!_generalOptions.IsDisableReversibleCheck() + && SkylarkUtil.ConvertFromNoneable(reversibleCheckObj, mode == WorkflowMode.ChangeRequest)) + { + try + { + reverseTransform = sequenceTransform.Reverse(); + } + catch (NonReversibleValidationException e) + { + throw StarlarkRt.Errorf("{0}", e.Message); + } + } + + var changeIdentity = GetChangeIdentity(changeIdentityObj); + + if (!StarlarkRt.IsNullOrNone(experimentalCustomRevIdField)) + { + _generalOptions.GetConsole() + .Warn("experimental_custom_rev_id is deprecated. Use custom_rev_id instead."); + } + + string? customRevId = SkylarkUtil.ConvertFromNoneable( + customRevIdField, SkylarkUtil.ConvertFromNoneable(experimentalCustomRevIdField, null)); + + SkylarkUtil.Check( + customRevId == null || CustomRevidFormat.IsMatch(customRevId), + "Invalid custom_rev_id format. Format: {0}", + CustomRevidFormat.ToString()); + + if (setRevId) + { + SkylarkUtil.Check( + mode != WorkflowMode.ChangeRequest || customRevId == null, + "custom_rev_id is not allowed to be used in CHANGE_REQUEST mode if set_rev_id is set" + + " to true. custom_rev_id is used for looking for the baseline in the origin. No" + + " revId is stored in the destination."); + } + else + { + SkylarkUtil.Check( + mode == WorkflowMode.ChangeRequest || mode == WorkflowMode.ChangeRequestFromSot, + "'set_rev_id = False' is only supported for CHANGE_REQUEST and" + + " CHANGE_REQUEST_FROM_SOT mode."); + } + + if (smartPrune) + { + SkylarkUtil.Check( + mode == WorkflowMode.ChangeRequest, + "'smart_prune = True' is only supported for CHANGE_REQUEST mode."); + } + + if (checkLastRevState) + { + SkylarkUtil.Check( + mode != WorkflowMode.ChangeRequest, + "{0} is not compatible with {1}", + CheckLastRevStateName, + WorkflowMode.ChangeRequest); + } + + var resolvedAuthoring = authoring; + var defaultAuthorFlag = _workflowOptions.GetDefaultAuthorFlag(); + if (defaultAuthorFlag != null) + { + resolvedAuthoring = + new AuthoringType(defaultAuthorFlag, authoring.GetMode(), authoring.GetAllowPredicate()); + } + + string? consistencyFilePath = SkylarkUtil.ConvertFromNoneable(consistencyFilePathObj, null); + MergeImportConfiguration? mergeImport; + if (mergeImportObj is bool objectValue) + { + mergeImport = + objectValue + ? MergeImportConfiguration.Create( + "", + Glob.AllFiles, + !string.IsNullOrEmpty(consistencyFilePath), + MergeImportConfiguration.MergeStrategy.DIFF3) + : null; + } + else + { + mergeImport = SkylarkUtil.ConvertFromNoneable(mergeImportObj, null); + } + + var consistencyConfig = ResolveConsistencyFileConfig(consistencyFileObj, consistencyFilePath); + + if (mergeImport != null && mergeImport.UseConsistencyFile()) + { + SkylarkUtil.Check( + consistencyConfig != null, + "error: use_consistency_file set but consistency_file.path is null"); + } + + if (consistencyConfig != null && mergeImport != null) + { + SkylarkUtil.Check( + mergeImport.UseConsistencyFile(), + "error: consistency_file.path set and merge import is enabled, but" + + " use_consistency_file in merge_import is false"); + } + + var autoPatchfileConfiguration = + SkylarkUtil.ConvertFromNoneable(autoPatchFileConfigurationObj, null); + + var effectiveMode = + _generalOptions.Squash || _workflowOptions.ImportSameVersion ? WorkflowMode.Squash : mode; + + // TODO(port): reconcile — Debug.getCallStack / captureDefinitionStackLocals depend on the + // Starlark Debug helper. Left as empty until reconciled with the Workflow port. + var locals = ImmutableArray>.Empty; + + // Mirrors Java's `new Workflow<>(...)` where origin/destination arrive as untyped Starlark + // objects. The non-generic factory reflects the concrete revision types out of origin and + // destination and instantiates the correctly-typed Workflow. + var workflow = + Copybara.Workflow.Create( + workflowName, + SkylarkUtil.ConvertFromNoneable(description, null), + origin, + destination, + resolvedAuthoring, + sequenceTransform, + _workflowOptions.GetLastRevision(), + _workflowOptions.IsInitHistory(), + _generalOptions, + Glob.WrapGlob(originFiles, Glob.AllFiles), + Glob.WrapGlob(destinationFiles, Glob.AllFiles), + effectiveMode, + _workflowOptions, + reverseTransform, + Glob.WrapGlob(reversibleCheckIgnoreFiles, null), + askForConfirmation, + _mainConfigFile, + _allConfigFiles, + dryRunMode, + checkLastRevState || _workflowOptions.CheckLastRevState, + ConvertListOfActions(afterMigrations, _printHandler), + ConvertListOfActions(afterAllMigrations, _printHandler), + changeIdentity, + setRevId, + smartPrune, + mergeImport, + autoPatchfileConfiguration, + AsSingleTransform(afterMergeTransformations), + _workflowOptions.MigrateNoopChanges || migrateNoopChanges, + customRevId, + checkout, + consistencyConfig, + _workflowOptions.ExpectedFixedRef, + _workflowOptions.PinnedFixedRef, + thread.GetCallStack(), + locals); + + var module = Module.OfInnermostEnclosingStarlarkFunction(thread)!; + RegisterGlobalMigration(workflowName, workflow, module); + } + + private Copybara.Transform.Sequence AsSingleTransform(StarlarkSequence transformations) => + Copybara.Transform.Sequence.FromConfig( + _generalOptions.Profiler(), + null, + _workflowOptions, + transformations, + "transformations", + _printHandler, + _debugOptions.TransformWrapper, + Copybara.Transform.Sequence.NoopBehavior.NOOP_IF_ANY_NOOP); + + private static ImmutableArray GetChangeIdentity(object changeIdentityObj) + { + string? changeIdentity = SkylarkUtil.ConvertFromNoneable(changeIdentityObj, null); + + if (changeIdentity == null) + { + return ImmutableArray.Empty; + } + + var result = new Parser().Parse(changeIdentity); + bool configVarFound = false; + foreach (var token in result) + { + if (token.GetTokenType() != TokenType.Interpolation) + { + continue; + } + + if (token.GetValue() == Copybara.Workflow.COPYBARA_CONFIG_PATH_IDENTITY_VAR) + { + configVarFound = true; + continue; + } + + if (token.GetValue() == Copybara.Workflow.COPYBARA_WORKFLOW_NAME_IDENTITY_VAR + || token.GetValue() == Copybara.Workflow.COPYBARA_REFERENCE_IDENTITY_VAR + || token.GetValue().StartsWith(Copybara.Workflow.COPYBARA_REFERENCE_LABEL_VAR)) + { + continue; + } + + throw StarlarkRt.Errorf("Unrecognized variable: {0}", token.GetValue()); + } + + SkylarkUtil.Check( + configVarFound, + "${{{0}}} variable is required", + Copybara.Workflow.COPYBARA_CONFIG_PATH_IDENTITY_VAR); + return result.ToImmutableArray(); + } + + [StarlarkMethod("move", + Doc = "Moves files between directories and renames files", + UseStarlarkThread = true)] + public ITransformation Move( + [Param(Name = "before", Named = true, + Doc = "The name of the file or directory before moving.")] + string before, + [Param(Name = "after", Named = true, + Doc = "The name of the file or directory after moving.")] + string after, + [Param(Name = "paths", Named = true, DefaultValue = "None", + Doc = "A glob expression relative to 'before' if it represents a directory.", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object paths, + [Param(Name = "overwrite", Named = true, DefaultValue = "False", + Doc = "Overwrite destination files if they already exist.")] + bool overwrite, + [Param(Name = "regex_groups", Named = true, Positional = false, DefaultValue = "{}", + Doc = "A set of named regexes that can be used to match part of the file name.")] + Dict regexes, + StarlarkThread thread) + { + SkylarkUtil.Check( + before != after, + "Moving from the same folder to the same folder is a noop. Remove the transformation."); + + return CopyOrMove.CreateMove( + before, + after, + SkylarkUtil.ConvertStringMap(regexes, "regex_groups"), + Glob.WrapGlob(paths, Glob.AllFiles), + overwrite, + thread.GetCallerLocation()); + } + + [StarlarkMethod("rename", + Doc = + "A transformation for renaming several filenames in the working directory. This is a" + + " simplified version of core.move() for just renaming filenames.", + UseStarlarkThread = true)] + public ITransformation Rename( + [Param(Name = "before", Named = true, Doc = "The filepath or suffix to change")] + string before, + [Param(Name = "after", Named = true, Doc = "A filepath or suffix to use as replacement")] + string after, + [Param(Name = "paths", Named = true, DefaultValue = "None", + Doc = "A glob expression relative to 'before' if it represents a directory.", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object paths, + [Param(Name = "overwrite", Named = true, DefaultValue = "False", + Doc = "Overwrite destination files if they already exist.")] + bool overwrite, + [Param(Name = "suffix", Named = true, DefaultValue = "False", + Doc = "When set to true, it will match partial parts of the path string.")] + bool suffix, + StarlarkThread thread) + { + SkylarkUtil.Check( + before != after, + "Renaming from the same filename to the same filename is a noop. Remove the" + + " transformation."); + + // TODO(port): reconcile — Rename transform is being ported concurrently. + return new Copybara.Transform.Rename( + before, + after, + Glob.WrapGlob(paths, Glob.AllFiles), + overwrite, + suffix, + thread.GetCallerLocation()); + } + + [StarlarkMethod("copy", + Doc = "Copy files between directories and renames files", + UseStarlarkThread = true)] + public ITransformation Copy( + [Param(Name = "before", Named = true, Doc = "The name of the file or directory to copy.")] + string before, + [Param(Name = "after", Named = true, Doc = "The name of the file or directory destination.")] + string after, + [Param(Name = "paths", Named = true, DefaultValue = "None", + Doc = "A glob expression relative to 'before' if it represents a directory.", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object paths, + [Param(Name = "overwrite", Named = true, DefaultValue = "False", + Doc = "Overwrite destination files if they already exist.")] + bool overwrite, + [Param(Name = "regex_groups", Named = true, Positional = false, DefaultValue = "{}", + Doc = "A set of named regexes that can be used to match part of the file name.")] + Dict regexes, + StarlarkThread thread) + { + SkylarkUtil.Check( + before != after, + "Copying from the same folder to the same folder is a noop. Remove the transformation."); + return CopyOrMove.CreateCopy( + before, + after, + SkylarkUtil.ConvertStringMap(regexes, "regex_groups"), + Glob.WrapGlob(paths, Glob.AllFiles), + overwrite, + thread.GetCallerLocation()); + } + + [StarlarkMethod("remove", + Doc = + "Remove files from the workdir. **This transformation is only meant to be used inside" + + " core.transform for reversing core.copy like transforms**.", + UseStarlarkThread = true)] + public Copybara.Transform.Remove Remove( + [Param(Name = "paths", Named = true, Doc = "The files to be deleted")] + Glob paths, + StarlarkThread thread) => + // TODO(port): reconcile — Remove transform is being ported concurrently. + new(paths, thread.GetCallerLocation()); + + [StarlarkMethod("convert_encoding", + Doc = "Change the encoding for a set of files", + UseStarlarkThread = true)] + public ITransformation ConvertEncoding( + [Param(Name = "before", Named = true, + Doc = "The expected encoding of the files before transformation.")] + string before, + [Param(Name = "after", Named = true, Doc = "The encoding to convert to.")] + string after, + [Param(Name = "paths", Named = true, Doc = "The files to be converted")] + Glob paths, + StarlarkThread thread) + { + System.Text.Encoding cBefore; + try + { + cBefore = System.Text.Encoding.GetEncoding(before); + } + catch (ArgumentException e) + { + throw new EvalException("Incorrect charset " + before + " for 'before': " + e.Message); + } + + System.Text.Encoding cAfter; + try + { + cAfter = System.Text.Encoding.GetEncoding(after); + } + catch (ArgumentException e) + { + throw new EvalException("Incorrect charset " + after + " for 'after': " + e.Message); + } + + // TODO(port): reconcile — ConvertEncoding transform is being ported concurrently. + return new Copybara.ConvertEncoding(cBefore, cAfter, paths); + } + + [StarlarkMethod("replace", + Doc = + "Replace a text with another text using optional regex groups. This transformation can" + + " be automatically reversed.", + UseStarlarkThread = true)] + public Replace Replace( + [Param(Name = "before", Named = true, + Doc = "The text before the transformation. Can contain references to regex groups.")] + string before, + [Param(Name = "after", Named = true, + Doc = "The text after the transformation.")] + string after, + [Param(Name = "regex_groups", Named = true, DefaultValue = "{}", + Doc = "A set of named regexes that can be used to match part of the replaced text.")] + Dict regexes, + [Param(Name = "paths", Named = true, DefaultValue = "None", + Doc = "A glob expression relative to the workdir representing the files to apply.", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object paths, + [Param(Name = "first_only", Named = true, DefaultValue = "False", + Doc = "If true, only replaces the first instance rather than all.")] + bool firstOnly, + [Param(Name = "multiline", Named = true, DefaultValue = "False", + Doc = "Whether to replace text that spans more than one line.")] + bool multiline, + [Param(Name = "repeated_groups", Named = true, DefaultValue = "False", + Doc = "Allow to use a group multiple times.")] + bool repeatedGroups, + [Param(Name = "ignore", Named = true, DefaultValue = "[]", + Doc = "A set of regexes to ignore lines/files.")] + StarlarkSequence ignore, + StarlarkThread thread) => + Copybara.Transform.Replace.Create( + thread.GetCallerLocation(), + before, + after, + SkylarkUtil.ConvertStringMap(regexes, "regex_groups"), + Glob.WrapGlob(paths, Glob.AllFiles), + firstOnly, + multiline, + repeatedGroups, + SkylarkUtil.ConvertStringList(ignore, "patterns_to_ignore"), + _workflowOptions); + + [StarlarkMethod("todo_replace", + Doc = "Replace Google style TODOs. For example `TODO(username, othername)`.", + UseStarlarkThread = true)] + public TodoReplace TodoReplace( + [Param(Name = "tags", Named = true, DefaultValue = "['TODO', 'NOTE']", + Doc = "Prefix tag to look for", + AllowedTypes = new[] { typeof(StarlarkSequence) })] + StarlarkSequence skyTags, + [Param(Name = "mapping", Named = true, DefaultValue = "{}", + Doc = "Mapping of users/strings")] + Dict skyMapping, + [Param(Name = "mode", Named = true, DefaultValue = "'MAP_OR_IGNORE'", + Doc = "Mode for the replace.")] + string modeStr, + [Param(Name = "paths", Named = true, DefaultValue = "None", + Doc = "A glob expression relative to the workdir representing the files to apply.", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object paths, + [Param(Name = "default", Named = true, DefaultValue = "None", + Doc = "Default value if mapping not found.", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object skyDefault, + [Param(Name = "ignore", Named = true, DefaultValue = "None", + Doc = "If set, elements within TODO that match the regex will be ignored.", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object regexToIgnore, + StarlarkThread thread) + { + var mode = SkylarkUtil.StringToEnum("mode", modeStr); + var mapping = SkylarkUtil.ConvertStringMap(skyMapping, "mapping"); + string? defaultString = SkylarkUtil.ConvertFromNoneable(skyDefault, null); + var tags = SkylarkUtil.ConvertStringList(skyTags, "tags").ToImmutableArray(); + string? ignorePattern = SkylarkUtil.ConvertFromNoneable(regexToIgnore, null); + Regex? regexIgnorelist = ignorePattern != null ? new Regex(ignorePattern) : null; + + SkylarkUtil.Check(tags.Length != 0, "'tags' cannot be empty"); + if (mode is Copybara.Transform.TodoReplace.Mode.MAP_OR_DEFAULT + or Copybara.Transform.TodoReplace.Mode.USE_DEFAULT) + { + SkylarkUtil.Check(defaultString != null, "'default' needs to be set for mode '{0}'", mode); + } + else + { + SkylarkUtil.Check(defaultString == null, "'default' cannot be used for mode '{0}'", mode); + } + + if (mode is Copybara.Transform.TodoReplace.Mode.USE_DEFAULT + or Copybara.Transform.TodoReplace.Mode.SCRUB_NAMES) + { + SkylarkUtil.Check(mapping.Count == 0, "'mapping' cannot be used with mode {0}", mode); + } + + return new Copybara.Transform.TodoReplace( + thread.GetCallerLocation(), + Glob.WrapGlob(paths, Glob.AllFiles), + tags, + mode, + mapping, + defaultString, + _workflowOptions.Parallelizer(), + regexIgnorelist); + } + + public const string TodoFilterReplaceExample = + "core.filter_replace(\n" + + " regex = 'TODO\\\\((.*?)\\\\)',\n" + + " group = 1,\n" + + " mapping = core.replace_mapper([\n" + + " core.replace(\n" + + " before = '${p}foo${s}',\n" + + " after = '${p}fooz${s}',\n" + + " regex_groups = { 'p': '.*', 's': '.*'}\n" + + " ),\n" + + " ],\n" + + " all = True\n" + + " )\n" + + ")"; + + public const string SimpleFilterReplaceExample = + "core.filter_replace(\n" + + " regex = 'a.*',\n" + + " mapping = {\n" + + " 'afoo': 'abar',\n" + + " 'abaz': 'abam'\n" + + " }\n" + + ")"; + + [StarlarkMethod("filter_replace", + Doc = + "Applies an initial filtering to find a substring to be replaced and then applies a" + + " `mapping` of replaces for the matched text.", + UseStarlarkThread = true)] + public FilterReplace FilterReplace( + [Param(Name = "regex", Named = true, Doc = "A re2 regex to match a substring of the file", + AllowedTypes = new[] { typeof(string) })] + string regex, + [Param(Name = "mapping", Named = true, DefaultValue = "{}", + Doc = "A mapping function like core.replace_mapper or a dict with mapping values.")] + object mapping, + [Param(Name = "group", Named = true, DefaultValue = "None", + Doc = "Extract a regex group from the matching text.", + AllowedTypes = new[] { typeof(StarlarkInt), typeof(NoneType) })] + object group, + [Param(Name = "paths", Named = true, DefaultValue = "None", + Doc = "A glob expression relative to the workdir representing the files to apply.", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object paths, + [Param(Name = "reverse", Named = true, DefaultValue = "None", + Doc = "A re2 regex used as reverse transformation", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object reverse, + StarlarkThread thread) + { + var func = GetMappingFunction(mapping); + + string afterPattern = SkylarkUtil.ConvertFromNoneable(reverse, regex); + int numGroup = SkylarkUtil.ConvertFromNoneable(group, StarlarkInt.Of(0)).ToInt("group"); + var beforeRegex = new Regex(regex); + int beforeGroups = beforeRegex.GetGroupNumbers().Length - 1; + SkylarkUtil.Check( + numGroup <= beforeGroups, + "group idx is greater than the number of groups defined in '{0}'. Regex has {1} groups", + beforeRegex.ToString(), + beforeGroups); + var afterRegex = new Regex(afterPattern); + int afterGroups = afterRegex.GetGroupNumbers().Length - 1; + SkylarkUtil.Check( + numGroup <= afterGroups, + "reverse_group idx is greater than the number of groups defined in '{0}'. Regex has {1}" + + " groups", + afterRegex.ToString(), + afterGroups); + return new FilterReplace( + _workflowOptions, + beforeRegex, + afterRegex, + numGroup, + numGroup, + func, + Glob.WrapGlob(paths, Glob.AllFiles), + thread.GetCallerLocation()); + } + + public static IReversibleFunction GetMappingFunction(object mapping) + { + if (mapping is Dict dict) + { + var map = SkylarkUtil.ConvertStringMap(dict, "mapping"); + SkylarkUtil.Check( + map.Count != 0, "Empty mapping is not allowed. Remove the transformation instead"); + return new MapMapper(map.ToImmutableDictionary()); + } + + SkylarkUtil.Check( + mapping is IReversibleFunction, + "mapping has to be instance of map or a reversible function"); + return (IReversibleFunction)mapping; + } + + [StarlarkMethod("replace_mapper", + Doc = + "A mapping function that applies a list of replaces until one replaces the text" + + " (Unless `all = True` is used).")] + public ReplaceMapper MapImports( + [Param(Name = "mapping", Named = true, + Doc = "The list of core.replace transformations", + AllowedTypes = new[] { typeof(StarlarkSequence) })] + StarlarkSequence mapping, + [Param(Name = "all", Named = true, Positional = false, DefaultValue = "False", + Doc = "Run all the mappings despite a replace happens.")] + bool all) + { + SkylarkUtil.Check(mapping.Count != 0, "Empty mapping is not allowed"); + var replaces = ImmutableArray.CreateBuilder(); + foreach (var obj in mapping) + { + if (obj is not ITransformation t) + { + throw StarlarkRt.Errorf("Expected a transformation in 'mapping'"); + } + + SkylarkUtil.Check( + t is Replace, "Only core.replace can be used as mapping, but got: {0}", t.Describe()); + var replace = (Replace)t; + SkylarkUtil.Check( + Equals(replace.GetPaths(), Glob.AllFiles), + "core.replace cannot use 'paths' inside core.replace_mapper"); + replaces.Add(replace); + } + + return new ReplaceMapper(replaces.ToImmutable(), all); + } + + [StarlarkMethod("verify_match", + Doc = + "Verifies that a RegEx matches (or not matches) the specified files. Does not transform" + + " anything, but will stop the workflow if it fails.", + UseStarlarkThread = true)] + public VerifyMatch VerifyMatch( + [Param(Name = "regex", Named = true, Doc = "The regex pattern to verify.")] + string regex, + [Param(Name = "paths", Named = true, DefaultValue = "None", + Doc = "A glob expression relative to the workdir representing the files to apply.", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object paths, + [Param(Name = "verify_no_match", Named = true, DefaultValue = "False", + Doc = "If true, the transformation will verify that the RegEx does not match.")] + bool verifyNoMatch, + [Param(Name = "also_on_reversal", Named = true, DefaultValue = "False", + Doc = "If true, the check will also apply on the reversal.")] + bool alsoOnReversal, + [Param(Name = "failure_message", Named = true, DefaultValue = "None", + Doc = "Optional string that will be included in the failure message.")] + object failureMessage, + StarlarkThread thread) => + Copybara.Transform.VerifyMatch.Create( + thread.GetCallerLocation(), + regex, + Glob.WrapGlob(paths, Glob.AllFiles), + verifyNoMatch, + alsoOnReversal, + SkylarkUtil.ConvertOptionalString(failureMessage), + _workflowOptions.Parallelizer()); + + [StarlarkMethod("transform", + Doc = + "Groups some transformations in a transformation that can contain a particular," + + " manually-specified, reversal.")] + public ITransformation Transform( + [Param(Name = "transformations", Named = true, + Doc = "The list of transformations to run as a result of running this transformation.", + AllowedTypes = new[] { typeof(StarlarkSequence) })] + StarlarkSequence transformations, + [Param(Name = "reversal", Named = true, Positional = false, DefaultValue = "None", + Doc = "The list of transformations to run in reverse.", + AllowedTypes = new[] { typeof(StarlarkSequence), typeof(NoneType) })] + object reversal, + [Param(Name = "name", Named = true, Positional = false, DefaultValue = "None", + Doc = "Optional string identifier to name this transform.")] + object name, + [Param(Name = "ignore_noop", Named = true, Positional = false, DefaultValue = "None", + Doc = "WARNING: Deprecated. Use `noop_behavior` instead.", + AllowedTypes = new[] { typeof(bool), typeof(NoneType) })] + object ignoreNoop, + [Param(Name = "noop_behavior", Named = true, Positional = false, DefaultValue = "None", + Doc = "How to handle no-op transformations.", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object noopBehaviorString) + { + ValidationException.CheckCondition( + StarlarkRt.IsNullOrNone(ignoreNoop) || StarlarkRt.IsNullOrNone(noopBehaviorString), + "The deprecated param 'ignore_noop' cannot be set simultaneously with 'noop_behavior'." + + " Prefer using 'noop_behavior'."); + var noopBehavior = + SkylarkUtil.StringToEnum( + "noop_behavior", + SkylarkUtil.ConvertFromNoneable(noopBehaviorString, "NOOP_IF_ANY_NOOP")); + if (ignoreNoop is true) + { + noopBehavior = Copybara.Transform.Sequence.NoopBehavior.IGNORE_NOOP; + } + else if (ignoreNoop is false) + { + noopBehavior = Copybara.Transform.Sequence.NoopBehavior.FAIL_IF_ANY_NOOP; + } + + string? convertedName = SkylarkUtil.ConvertFromNoneable(name, null); + if (!_transformNames.TryGetValue(_mainConfigFile, out var names)) + { + names = new HashSet(); + _transformNames[_mainConfigFile] = names; + } + + if (convertedName != null && !names.Add(convertedName)) + { + throw new ValidationException($"Name `{convertedName}` already used."); + } + + var forward = + Copybara.Transform.Sequence.FromConfig( + _generalOptions.Profiler(), + convertedName, + _workflowOptions, + transformations, + "transformations", + _printHandler, + _debugOptions.TransformWrapper, + noopBehavior); + StarlarkSequence? reverseList = SkylarkUtil.ConvertFromNoneable(reversal, null); + if (reverseList == null) + { + try + { + reverseList = StarlarkList.ImmutableCopyOf(new object?[] { forward.Reverse() }); + } + catch (NonReversibleValidationException) + { + throw StarlarkRt.Errorf( + "transformations are not automatically reversible. Use 'reversal' field to" + + " explicitly configure the reversal of the transform"); + } + } + + var reverse = + Copybara.Transform.Sequence.FromConfig( + _generalOptions.Profiler(), + convertedName, + _workflowOptions, + reverseList, + "reversal", + _printHandler, + _debugOptions.TransformWrapper, + noopBehavior); + return new ExplicitReversal(forward, reverse); + } + + [StarlarkMethod("dynamic_transform", + Doc = "Create a dynamic Starlark transformation. This should only be used by library developers", + UseStarlarkThread = true)] + public ITransformation DynamicTransform( + [Param(Name = "impl", Named = true, Doc = "The Starlark function to call")] + IStarlarkCallable impl, + [Param(Name = "params", Named = true, DefaultValue = "{}", + Doc = "The parameters to the function. Will be available under ctx.params")] + Dict @params, + StarlarkThread thread) => + // TODO(port): reconcile — SkylarkTransformation is being ported concurrently. + new SkylarkTransformation(impl, Dict.CopyOf(thread.Mutability, @params.Entries), _printHandler); + + [StarlarkMethod("dynamic_feedback", + Doc = "Create a dynamic Starlark feedback migration. This should only be used by library developers", + UseStarlarkThread = true)] + public Copybara.Action.IAction DynamicFeedback( + [Param(Name = "impl", Named = true, Doc = "The Starlark function to call")] + IStarlarkCallable impl, + [Param(Name = "params", Named = true, DefaultValue = "{}", + Doc = "The parameters to the function. Will be available under ctx.params")] + Dict @params, + StarlarkThread thread) => + // TODO(port): reconcile — StarlarkAction is being ported concurrently. + new Copybara.Action.StarlarkAction( + FindCallableName(impl, thread), impl, Dict.CopyOf(thread.Mutability, @params.Entries), + _printHandler); + + [StarlarkMethod("action", + Doc = "Create a dynamic Starlark action. This should only be used by library developers.", + UseStarlarkThread = true)] + public Copybara.Action.IAction Action( + [Param(Name = "impl", Named = true, Doc = "The Starlark function to call")] + IStarlarkCallable impl, + [Param(Name = "params", Named = true, DefaultValue = "{}", + Doc = "The parameters to the function. Will be available under ctx.params")] + Dict @params, + StarlarkThread thread) => + new Copybara.Action.StarlarkAction( + FindCallableName(impl, thread), impl, Dict.CopyOf(thread.Mutability, @params.Entries), + _printHandler); + + private static string FindCallableName(IStarlarkCallable impl, StarlarkThread thread) + { + string name = impl.Name; + var stack = thread.GetCallStack(); + if (name == "lambda" && stack.Length > 1 + && stack[stack.Length - 2].Name != "") + { + name = stack[stack.Length - 2].Name; + } + + return name; + } + + [StarlarkMethod("fail_with_noop", + Doc = "If invoked, it will fail the current migration as a noop")] + public Copybara.Action.IAction FailWithNoop( + [Param(Name = "msg", Named = true, Doc = "The noop message")] + string msg) => + throw new EmptyChangeException(msg); + + [StarlarkMethod("main_config_path", + Doc = "Location of the config file. This is subject to change", + StructField = true)] + public string GetMainConfigFile() => _mainConfigFile.GetIdentifier(); + + [StarlarkMethod("feedback", + Doc = + "Defines a migration of changes' metadata, that can be invoked via the Copybara command" + + " in the same way as a regular workflow migrates the change itself.", + UseStarlarkThread = true)] + public NoneType Feedback( + [Param(Name = "name", Named = true, Positional = false, Doc = "The name of the feedback workflow.")] + string workflowName, + [Param(Name = "origin", Named = true, Positional = false, Doc = "The trigger of a feedback migration.")] + ITrigger trigger, + [Param(Name = "destination", Named = true, Positional = false, Doc = "Where to write change metadata to.")] + object destination, + [Param(Name = "actions", Named = true, Positional = false, DefaultValue = "[]", + Doc = "DEPRECATED: **DO NOT USE**")] + StarlarkSequence actionList, + [Param(Name = "action", Named = true, Positional = false, DefaultValue = "None", + Doc = "An action to execute when the migration is triggered")] + object action, + [Param(Name = "description", Named = true, Positional = false, DefaultValue = "None", + Doc = "A description of what this workflow achieves", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object description, + StarlarkThread thread) + { + var destinationProvider = (Copybara.IEndpointProvider)destination; + var migration = + new ActionMigration( + workflowName, + SkylarkUtil.ConvertFromNoneable(description, null), + _mainConfigFile, + trigger, + new StructImpl( + ImmutableDictionary.Empty.Add( + "destination", destinationProvider.GetEndpoint())), + HandleActionActionsMigration(actionList, action), + _generalOptions, + "feedback", + fileSystem: false, + thread.GetCallStack()); + var module = Module.OfInnermostEnclosingStarlarkFunction(thread)!; + RegisterGlobalMigration(workflowName, migration, module); + return StarlarkRt.None; + } + + private ImmutableArray HandleActionActionsMigration( + StarlarkSequence actionList, object action) + { + if (actionList.Count == 0 && ReferenceEquals(action, StarlarkRt.None)) + { + throw new EvalException("'action' is a required field"); + } + + if (actionList.Count != 0 && !ReferenceEquals(action, StarlarkRt.None)) + { + throw new EvalException( + "Cannot use both 'action' and 'actions' field. 'actions' is deprecated, so use" + + " 'action'"); + } + + if (!ReferenceEquals(action, StarlarkRt.None)) + { + return ImmutableArray.Create(MaybeWrapAction(_printHandler, action)); + } + + return ConvertListOfActions(actionList, _printHandler); + } + + [StarlarkMethod("action_migration", + Doc = + "Defines a migration that is more flexible/less-opinionated migration than" + + " `core.workflow`.", + UseStarlarkThread = true)] + public NoneType ActionMigrationMethod( + [Param(Name = "name", Named = true, Positional = false, Doc = "The name of the migration.")] + string workflowName, + [Param(Name = "origin", Named = true, Positional = false, + Doc = "The trigger endpoint of the migration. Accessible as `ctx.origin`")] + ITrigger trigger, + [Param(Name = "endpoints", Named = true, Positional = false, + Doc = "One or more endpoints that the migration will have access to.")] + IStructure endpoints, + [Param(Name = "action", Named = true, Positional = false, + Doc = "The action to execute when the migration is triggered.")] + object action, + [Param(Name = "description", Named = true, Positional = false, DefaultValue = "None", + Doc = "A description of what this workflow achieves", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object description, + [Param(Name = "filesystem", Named = true, Positional = false, DefaultValue = "False", + Doc = "If true, the migration provides access to the filesystem to the endpoints", + AllowedTypes = new[] { typeof(bool) })] + bool filesystem, + StarlarkThread thread) + { + var actions = ImmutableArray.Create(MaybeWrapAction(_printHandler, action)); + var endpointsMap = GetEndpoints(endpoints); + if (!endpointsMap.ContainsKey(ActionMigration.DestinationEndpointName)) + { + throw new EvalException("Action migration must have an endpoint named 'destination'."); + } + + var migration = + new ActionMigration( + workflowName, + SkylarkUtil.ConvertFromNoneable(description, null), + _mainConfigFile, + trigger, + new StructImpl(endpointsMap), + actions, + _generalOptions, + "action_migration", + filesystem, + thread.GetCallStack()); + var module = Module.OfInnermostEnclosingStarlarkFunction(thread)!; + RegisterGlobalMigration(workflowName, migration, module); + return StarlarkRt.None; + } + + private static ImmutableDictionary GetEndpoints(IStructure endpoints) + { + var result = ImmutableDictionary.CreateBuilder(); + foreach (var fieldName in endpoints.GetFieldNames()) + { + var epProvider = endpoints.GetValue(fieldName); + SkylarkUtil.Check( + epProvider is Copybara.IEndpointProvider, + "Only endpoints can be used as values in 'endpoints' but got type '{0}' for {1}", + StarlarkRt.Type(epProvider), + fieldName); + result[fieldName] = ((Copybara.IEndpointProvider)epProvider!).GetEndpoint(); + } + + return result.ToImmutable(); + } + + [StarlarkMethod("console", + StructField = true, + Doc = "Returns a handle to the console object.")] + public SkylarkConsole Console() + { + lock (_consoleLock) + { + _console ??= new SkylarkConsole(_generalOptions.GetConsole()); + } + + return _console; + } + + /// Registers a in the global registry. + protected void RegisterGlobalMigration(string name, IMigration migration, Module module) => + GlobalMigrations.GetGlobalMigrations(module).AddMigration(name, migration); + + [StarlarkMethod("format", + Doc = "Formats a String using Java's String.format style.")] + public string Format( + [Param(Name = "format", Named = true, Doc = "The format string")] + string format, + [Param(Name = "args", Named = true, Doc = "The arguments to format")] + StarlarkSequence args) + { + // Convert StarlarkInt to types known to the formatter. + var array = args.ToArray(); + for (int i = 0; i < array.Length; i++) + { + if (array[i] is StarlarkInt si) + { + array[i] = si.ToNumber(); + } + } + + try + { + // NOTE(port): upstream uses Java's String.format ('%s'); this port uses .NET composite + // formatting ('{0}'). Accepted deviation — see CLAUDE.md. + return string.Format(format, array); + } + catch (FormatException e) + { + throw StarlarkRt.Errorf("Invalid format: {0}: {1}", format, e.Message); + } + } + + // TODO(port): reconcile — the Copybara.Version namespace (IVersionSelector, + // CustomVersionSelector, LatestVersionSelector, OrderedVersionSelector, + // RequestedVersionSelector) is being ported by another agent; these two methods forward-reference + // it and will compile once that scope lands. + [StarlarkMethod("custom_version_selector", + Doc = + "This is experimental: Custom version selector, users are able to define their own" + + " sorting comparator and filter candidates by regex.", + UseStarlarkThread = true)] + public Copybara.Version.IVersionSelector CustomVersionSelector( + [Param(Name = "comparator", Named = true, + Doc = "A callable comparator of two strings.", + AllowedTypes = new[] { typeof(IStarlarkCallable) })] + IStarlarkCallable comparator, + [Param(Name = "regex_filter", Named = true, DefaultValue = "None", + Doc = "Only versions that match this regex will be considered.", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object rawRegexFilter, + StarlarkThread thread) + { + string? filterByRegex = SkylarkUtil.ConvertFromNoneable(rawRegexFilter, null); + // TODO(port): reconcile — CustomVersionSelector is being ported concurrently. + return new Copybara.Version.CustomVersionSelector(comparator, filterByRegex); + } + + [StarlarkMethod("latest_version", + Doc = + "Selects the latest version that matches the format. Using --force in the CLI will force" + + " to use the reference passed as argument instead.", + UseStarlarkThread = true)] + public Copybara.Version.IVersionSelector VersionSelector( + [Param(Name = "format", Named = true, Doc = "The format of the version.")] + string regex, + [Param(Name = "regex_groups", Named = true, DefaultValue = "{}", + Doc = "A set of named regexes that can be used to match part of the versions.")] + Dict groups, + StarlarkThread thread) + { + var groupsMap = SkylarkUtil.ConvertStringMap(groups, "regex_groups"); + + var elements = new SortedDictionary(); + var regexKey = new Regex("^([sn])([0-9])$"); + foreach (var s in groupsMap.Keys) + { + var matcher = regexKey.Match(s); + SkylarkUtil.Check( + matcher.Success, + "Incorrect key for regex_group. Should be in the format of n0, n1, etc. or s0, s1," + + " etc. Value: {0}", + s); + var type = matcher.Groups[1].Value == "s" + ? Copybara.Version.LatestVersionSelector.VersionElementType.ALPHABETIC + : Copybara.Version.LatestVersionSelector.VersionElementType.NUMERIC; + int num = int.Parse(matcher.Groups[2].Value); + SkylarkUtil.Check( + !elements.ContainsKey(num) || elements[num] == type, + "Cannot use same n in both s{0} and n{1}: {2}", + num, + num, + s); + elements[num] = type; + } + + foreach (var num in elements.Keys) + { + if (num > 0) + { + SkylarkUtil.Check( + elements.ContainsKey(num - 1), + "Cannot have s{0} or n{1} if s{2} or n{3} doesn't exist", + num, + num, + num - 1, + num - 1); + } + } + + // TODO(port): reconcile — version selector types are being ported concurrently. + var versionPicker = new Copybara.Version.LatestVersionSelector( + regex, Copybara.Transform.Replace.ParsePatterns(groupsMap), elements, thread.GetCallerLocation()); + var extraGroups = versionPicker.GetUnmatchedGroups(); + SkylarkUtil.Check( + extraGroups.Count == 0, "Extra regex_groups not used in pattern: {0}", extraGroups); + if (_generalOptions.IsForced() || _generalOptions.IsVersionSelectorUseCliRef()) + { + return new Copybara.Version.OrderedVersionSelector( + ImmutableArray.Create( + new Copybara.Version.RequestedVersionSelector(), versionPicker)); + } + + return versionPicker; + } + + private static ImmutableArray ConvertListOfActions( + StarlarkSequence feedbackActions, StarlarkThread.PrintHandler? printHandler) + { + var actions = ImmutableArray.CreateBuilder(); + foreach (var action in feedbackActions) + { + actions.Add(MaybeWrapAction(printHandler, action!)); + } + + return actions.ToImmutable(); + } + + private static Copybara.Action.IAction MaybeWrapAction( + StarlarkThread.PrintHandler? printHandler, object action) + { + if (action is IStarlarkCallable callable) + { + return new Copybara.Action.StarlarkAction( + callable.Name, callable, Dict.Empty(), printHandler); + } + + if (action is Copybara.Action.IAction a) + { + return a; + } + + throw StarlarkRt.Errorf("Invalid action '{0}' of type: {1}", action, action.GetType()); + } + + public void SetConfigFile(ConfigFile mainConfigFile, ConfigFile currentConfigFile) => + _mainConfigFile = mainConfigFile; + + public void SetAllConfigResources(Func> allConfigFiles) => + _allConfigFiles = allConfigFiles; + + public void SetPrintHandler(StarlarkThread.PrintHandler printHandler) => + _printHandler = printHandler; + + [StarlarkMethod("merge_import_config", + Doc = "Describes which paths merge_import mode should be applied")] + public MergeImportConfiguration MergeImportConfigurationMethod( + [Param(Name = "package_path", Named = true, Positional = false, + Doc = "Package location (ex. 'google3/third_party/java/foo').")] + string packagePath, + [Param(Name = "paths", Named = true, Positional = false, DefaultValue = "None", + Doc = "Glob of paths to apply merge_import mode, relative to package_path", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object pathsObj, + [Param(Name = "use_consistency_file", Named = true, Positional = false, DefaultValue = "False", + Doc = "Deprecated. Use consistency_file in core.workflow instead.")] + bool useConsistencyFile, + [Param(Name = "merge_strategy", Named = true, Positional = false, DefaultValue = "'DIFF3'", + Doc = "The strategy to use for merging files.")] + string mergeStrategy) + { + var paths = Glob.WrapGlob(pathsObj, Glob.AllFiles); + return MergeImportConfiguration.Create( + packagePath, + paths!, + useConsistencyFile, + Enum.Parse(mergeStrategy)); + } + + [StarlarkMethod("consistency_file_config", + Doc = "Describes the configuration for consistency file options")] + public ConsistencyFileConfiguration ConsistencyFileConfig( + [Param(Name = "path", Named = true, Positional = false, + DefaultValue = "\"do-not-edit.bara.consistency\"", + Doc = "The path to the consistency file. Must end with .bara.consistency.", + AllowedTypes = new[] { typeof(string) })] + string path, + [Param(Name = "exclude_build_files", Named = true, Positional = false, DefaultValue = "False", + Doc = "Exclude BUILD files from being hashed in consistency files.")] + bool excludeBuildFiles) + { + try + { + ValidationException.CheckCondition( + path.EndsWith(".bara.consistency"), + "Consistency file path must end with .bara.consistency"); + } + catch (ValidationException e) + { + throw new EvalException(e.Message); + } + + return ConsistencyFileConfiguration.Create(path, excludeBuildFiles); + } + + [StarlarkMethod("autopatch_config", + Doc = "Describes in the configuration for automatic patch file generation")] + public AutoPatchfileConfiguration AutoPatchfileConfigurationMethod( + [Param(Name = "header", Named = true, Positional = false, DefaultValue = "None", + Doc = "A string to include at the beginning of each patch file", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object fileContentsPrefix, + [Param(Name = "suffix", Named = true, Positional = false, DefaultValue = "'.patch'", + Doc = "Suffix to use when saving patch files")] + string suffix, + [Param(Name = "directory_prefix", Named = true, Positional = false, DefaultValue = "''", + Doc = "Directory prefix used to relativize filenames when writing patch files.", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object directoryPrefix, + [Param(Name = "directory", Named = true, Positional = false, DefaultValue = "'AUTOPATCHES'", + Doc = "Directory in which to save the patch files.", + AllowedTypes = new[] { typeof(string), typeof(NoneType) })] + object directory, + [Param(Name = "strip_file_names_and_line_numbers", Named = true, Positional = false, + DefaultValue = "False", + Doc = "When true, strip filenames and line numbers from patch files")] + bool stripFileNamesAndLineNumbers, + [Param(Name = "strip_file_names", Named = true, Positional = false, DefaultValue = "False", + Doc = "When true, strip filenames from patch files")] + bool stripFileNames, + [Param(Name = "strip_line_numbers", Named = true, Positional = false, DefaultValue = "False", + Doc = "When true, strip line numbers from patch files")] + bool stripLineNumbers, + [Param(Name = "paths", Named = true, Positional = false, DefaultValue = "None", + Doc = "Only create patch files that match glob. Default is to match all files", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) })] + object globObj) + { + var glob = Glob.WrapGlob(globObj, Glob.AllFiles); + + if (stripFileNamesAndLineNumbers && (stripFileNames || stripLineNumbers)) + { + throw StarlarkRt.Errorf( + "Cannot set both strip_file_names_and_line_numbers and strip_file_names /" + + " strip_line_numbers"); + } + + if (stripFileNamesAndLineNumbers) + { + stripFileNames = true; + stripLineNumbers = true; + } + + return AutoPatchfileConfiguration.Create( + SkylarkUtil.ConvertFromNoneable(fileContentsPrefix, null)!, + suffix, + SkylarkUtil.ConvertFromNoneable(directoryPrefix, null)!, + SkylarkUtil.ConvertFromNoneable(directory, null), + stripFileNames, + stripLineNumbers, + glob!); + } + + private ConsistencyFileConfiguration? ResolveConsistencyFileConfig( + object consistencyFileObj, string? consistencyFilePath) + { + ConsistencyFileConfiguration? consistencyConfig = null; + object? consistencyFileVal = SkylarkUtil.ConvertFromNoneable(consistencyFileObj, null); + if (consistencyFileVal is bool b) + { + consistencyConfig = + b ? ConsistencyFileConfiguration.Create("do-not-edit.bara.consistency", false) : null; + } + else if (consistencyFileVal is ConsistencyFileConfiguration consistencyFileConfiguration) + { + consistencyConfig = consistencyFileConfiguration; + } + + // Validation for mutual exclusivity. + if (consistencyFilePath != null && consistencyFileVal != null) + { + throw StarlarkRt.Errorf( + "Cannot use both 'consistency_file_path' and 'consistency_file' parameters in" + + " workflow."); + } + + if (consistencyFilePath != null) + { + consistencyConfig = ConsistencyFileConfiguration.Create(consistencyFilePath, false); + } + + return consistencyConfig; + } +} diff --git a/src/Copybara.Core/Credentials/ConstantCredentialIssuer.cs b/src/Copybara.Core/Credentials/ConstantCredentialIssuer.cs new file mode 100644 index 000000000..699913233 --- /dev/null +++ b/src/Copybara.Core/Credentials/ConstantCredentialIssuer.cs @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2023 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; + +namespace Copybara.Credentials; + +/// A static CredentialIssuer, e.g. a password, username, api key, etc. +public class ConstantCredentialIssuer : CredentialIssuer +{ + private readonly string _secret; + private readonly string _name; + private readonly bool _open; + + public static ConstantCredentialIssuer CreateConstantSecret(string name, string secret) => + new(Preconditions.CheckNotNull(name), Preconditions.CheckNotNull(secret), false); + + public static ConstantCredentialIssuer CreateConstantOpenValue(string value) => + new(Preconditions.CheckNotNull(value), value, true); + + private ConstantCredentialIssuer(string name, string secret, bool open) + { + _secret = secret; + _name = name; + _open = open; + } + + public Credential Issue() => + _open ? new OpenCredential(_secret) : new StaticSecret(_name, _secret); + + public ImmutableSetMultimap Describe() => + ImmutableSetMultimap.CreateBuilder() + .Put("type", "constant") + .Put("name", _name) + .Put("open", _open ? "true" : "false") + .Build(); +} diff --git a/src/Copybara.Core/Credentials/Credential.cs b/src/Copybara.Core/Credentials/Credential.cs new file mode 100644 index 000000000..98ffbc339 --- /dev/null +++ b/src/Copybara.Core/Credentials/Credential.cs @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2023 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Credentials; + +/// Holder for a credential. +public interface Credential +{ + /// A safe value that describes the credential. + string PrintableValue(); + + /// Whether the credential is still believed to be valid. + bool Valid(); + + /// The raw secret, this should not be used outside of framework code. + /// if the secret cannot be retrieved. + string ProvideSecret(); +} diff --git a/src/Copybara.Core/Credentials/CredentialIssuer.cs b/src/Copybara.Core/Credentials/CredentialIssuer.cs new file mode 100644 index 000000000..d91158d96 --- /dev/null +++ b/src/Copybara.Core/Credentials/CredentialIssuer.cs @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2023 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Starlark.Eval; + +namespace Copybara.Credentials; + +/// +/// An object able to mint credentials. The issuer should handle caching etc. +/// +public interface CredentialIssuer : IStarlarkValue +{ + /// Issue a to be used by an endpoint. + /// if minting the credential fails. + Credential Issue(); + + /// Metadata describing this issuer. + ImmutableSetMultimap Describe(); +} diff --git a/src/Copybara.Core/Credentials/CredentialIssuingException.cs b/src/Copybara.Core/Credentials/CredentialIssuingException.cs new file mode 100644 index 000000000..e44c6072e --- /dev/null +++ b/src/Copybara.Core/Credentials/CredentialIssuingException.cs @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2023 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Exceptions; + +namespace Copybara.Credentials; + +/// An exception thrown if minting a credential fails. +public class CredentialIssuingException : ValidationException +{ + public CredentialIssuingException(string message) + : base(message) + { + } + + public CredentialIssuingException(string message, Exception? cause) + : base(message, cause) + { + } +} diff --git a/src/Copybara.Core/Credentials/CredentialModule.cs b/src/Copybara.Core/Credentials/CredentialModule.cs new file mode 100644 index 000000000..1d5984fb8 --- /dev/null +++ b/src/Copybara.Core/Credentials/CredentialModule.cs @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2023 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Exceptions; +using Starlark.Annot; +using Starlark.Eval; +using ConsoleImpl = Copybara.Util.Console.Console; + +namespace Copybara.Credentials; + +/// Starlark builtins to handle credentials. +[StarlarkBuiltin("credentials", Doc = "Module for working with credentials.")] +public class CredentialModule : IStarlarkValue +{ + protected readonly CredentialOptions Options; + protected readonly ConsoleImpl Console; + + public CredentialModule(ConsoleImpl console, CredentialOptions options) + { + Console = console; + Options = options; + } + + [StarlarkMethod( + "static_secret", + Doc = "Holder for secrets that can be in plaintext within the config.")] + public CredentialIssuer StaticSecret( + [Param(Name = "name", Doc = "A name for this secret.")] string name, + [Param(Name = "secret", Doc = "The secret value.")] string secret) + { + return ConstantCredentialIssuer.CreateConstantSecret(name, secret); + } + + [StarlarkMethod( + "static_value", + Doc = "Holder for credentials that are safe to read/log (e.g. 'x-access-token') .")] + public CredentialIssuer StaticValue( + [Param(Name = "value", Doc = "The open value.")] string value) + { + return ConstantCredentialIssuer.CreateConstantOpenValue(value); + } + + [StarlarkMethod( + "toml_key_source", + Doc = + "Supply an authentication credential from the " + + "file pointed to by the --http-credential-file flag.")] + public CredentialIssuer TomlKeySource( + [Param( + Name = "dot_path", + Doc = "Dot path to the data field containing the credential.", + AllowedTypes = new[] { typeof(string) })] + string dotPath) + { + if (Options.CredentialFile == null) + { + throw new ValidationException( + "Credential file for toml key source has not been supplied"); + } + + return new TomlKeySource(Options.CredentialFile, dotPath); + } + + [StarlarkMethod( + "username_password", + Doc = "A pair of username and password credential issuers.")] + public UsernamePasswordIssuer UsernamePassword( + [Param( + Name = "username", + Doc = "Username credential.", + AllowedTypes = new[] { typeof(CredentialIssuer) })] + CredentialIssuer username, + [Param( + Name = "password", + Doc = "Password credential.", + AllowedTypes = new[] { typeof(CredentialIssuer) })] + CredentialIssuer password) + { + return Credentials.UsernamePasswordIssuer.Create(username, password); + } +} diff --git a/src/Copybara.Core/Credentials/CredentialOptions.cs b/src/Copybara.Core/Credentials/CredentialOptions.cs new file mode 100644 index 000000000..dc421685b --- /dev/null +++ b/src/Copybara.Core/Credentials/CredentialOptions.cs @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2023 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Credentials; + +/// Flags related to credentials. +public class CredentialOptions : IOption +{ + /* + TODO(b/270712326) enable this flag + [Parameter( + Names = "--credential-file", + Description = "location of toml file for reading credentials")] + */ + + /// Location of the toml file for reading credentials. + public string? CredentialFile { get; set; } +} diff --git a/src/Copybara.Core/Credentials/CredentialRetrievalException.cs b/src/Copybara.Core/Credentials/CredentialRetrievalException.cs new file mode 100644 index 000000000..a41a7d95b --- /dev/null +++ b/src/Copybara.Core/Credentials/CredentialRetrievalException.cs @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2023 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Exceptions; + +namespace Copybara.Credentials; + +/// An exception thrown if retrieving a credential fails. +public class CredentialRetrievalException : ValidationException +{ + public CredentialRetrievalException(string message) + : base(message) + { + } + + public CredentialRetrievalException(string message, Exception? cause) + : base(message, cause) + { + } +} diff --git a/src/Copybara.Core/Credentials/OpenCredential.cs b/src/Copybara.Core/Credentials/OpenCredential.cs new file mode 100644 index 000000000..82d7b4f6e --- /dev/null +++ b/src/Copybara.Core/Credentials/OpenCredential.cs @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; + +namespace Copybara.Credentials; + +/// A non-secret Credential, e.g. a system username like "x-access-token". +public class OpenCredential : Credential +{ + private readonly string _open; + + public OpenCredential(string open) + { + _open = Preconditions.CheckNotNull(open); + } + + public string PrintableValue() => _open; + + public bool Valid() => true; + + public string ProvideSecret() => _open; + + public override string ToString() => PrintableValue(); +} diff --git a/src/Copybara.Core/Credentials/StaticSecret.cs b/src/Copybara.Core/Credentials/StaticSecret.cs new file mode 100644 index 000000000..585e77c41 --- /dev/null +++ b/src/Copybara.Core/Credentials/StaticSecret.cs @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2023 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; + +namespace Copybara.Credentials; + +/// A static secret Credential, e.g. a password, api key, etc. +public class StaticSecret : Credential +{ + private readonly string _secret; + protected readonly string Name; + + public StaticSecret(string name, string secret) + { + _secret = Preconditions.CheckNotNull(secret); + Name = Preconditions.CheckNotNull(name); + } + + public virtual string PrintableValue() => $""; + + public virtual bool Valid() => true; + + public virtual string ProvideSecret() => _secret; + + public override string ToString() => PrintableValue(); +} diff --git a/src/Copybara.Core/Credentials/TomlKeySource.cs b/src/Copybara.Core/Credentials/TomlKeySource.cs new file mode 100644 index 000000000..36c66d19e --- /dev/null +++ b/src/Copybara.Core/Credentials/TomlKeySource.cs @@ -0,0 +1,241 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text; +using Copybara.Common; +using Starlark.Eval; + +namespace Copybara.Credentials; + +/// Fetches a value located within a toml file. +/// +/// The upstream implementation uses the org.tomlj library. This port ships a minimal, +/// self-contained TOML reader that supports the subset of TOML needed to resolve a dotted key +/// path to a string value (top-level and [table] sections, dotted keys, basic and literal +/// strings). It is not a full TOML parser. +/// +public class TomlKeySource : CredentialIssuer, IStarlarkValue +{ + private readonly string _file; + private readonly string _dotPath; + + public TomlKeySource(string file, string keyPath) + { + _file = file; + _dotPath = keyPath; + } + + public Credential Issue() + { + IReadOnlyDictionary values; + try + { + values = ParseStringValues(File.ReadAllText(_file)); + } + catch (IOException e) + { + throw new CredentialIssuingException("Error reading Toml file.", e); + } + + if (!values.TryGetValue(_dotPath, out var data)) + { + throw new CredentialIssuingException( + string.Format("key {0} not found in file {1}", _dotPath, _file)); + } + + return new StaticSecret(_dotPath, data); + } + + public ImmutableSetMultimap Describe() => + ImmutableSetMultimap.CreateBuilder() + .Put("type", "Toml") + .Put("dotPath", _dotPath) + .Build(); + + /// + /// Parses the string-valued entries of a TOML document into a flat map keyed by fully-qualified + /// dotted path (e.g. foo.bar.baz). + /// + private static IReadOnlyDictionary ParseStringValues(string content) + { + var result = new Dictionary(); + string prefix = ""; + + foreach (var rawLine in content.Split('\n')) + { + var line = StripComment(rawLine).Trim(); + if (line.Length == 0) + { + continue; + } + + if (line[0] == '[') + { + // Table header: [a.b.c] or [[a.b]] (array of tables — treated like a table header). + int end = line.IndexOf(']'); + if (end < 0) + { + continue; + } + + var header = line.Substring(1, end - 1).Trim(); + if (header.StartsWith("[") && header.EndsWith("]")) + { + header = header.Substring(1, header.Length - 2).Trim(); + } + + prefix = JoinKeyParts(header); + continue; + } + + int eq = line.IndexOf('='); + if (eq < 0) + { + continue; + } + + var key = line.Substring(0, eq).Trim(); + var valueText = line.Substring(eq + 1).Trim(); + + if (!TryParseString(valueText, out var value)) + { + continue; + } + + var fullKey = JoinKeyParts(key); + if (prefix.Length > 0) + { + fullKey = prefix + "." + fullKey; + } + + result[fullKey] = value; + } + + return result; + } + + private static string JoinKeyParts(string key) + { + // Normalize dotted keys, stripping quotes and surrounding whitespace from each part. + var parts = key.Split('.'); + var builder = new StringBuilder(); + for (var i = 0; i < parts.Length; i++) + { + if (i > 0) + { + builder.Append('.'); + } + + builder.Append(Unquote(parts[i].Trim())); + } + + return builder.ToString(); + } + + private static string Unquote(string s) + { + if (s.Length >= 2 && + ((s[0] == '"' && s[^1] == '"') || (s[0] == '\'' && s[^1] == '\''))) + { + return s.Substring(1, s.Length - 2); + } + + return s; + } + + private static bool TryParseString(string valueText, out string value) + { + value = ""; + if (valueText.Length < 2) + { + return false; + } + + char quote = valueText[0]; + if (quote == '"' && valueText[^1] == '"') + { + value = Unescape(valueText.Substring(1, valueText.Length - 2)); + return true; + } + + if (quote == '\'' && valueText[^1] == '\'') + { + // Literal string: no escaping. + value = valueText.Substring(1, valueText.Length - 2); + return true; + } + + return false; + } + + private static string Unescape(string s) + { + if (s.IndexOf('\\') < 0) + { + return s; + } + + var builder = new StringBuilder(s.Length); + for (var i = 0; i < s.Length; i++) + { + var c = s[i]; + if (c == '\\' && i + 1 < s.Length) + { + var next = s[++i]; + builder.Append(next switch + { + 'n' => '\n', + 't' => '\t', + 'r' => '\r', + '"' => '"', + '\\' => '\\', + _ => next, + }); + } + else + { + builder.Append(c); + } + } + + return builder.ToString(); + } + + private static string StripComment(string line) + { + // Removes a trailing '#' comment, honoring quoted strings. + bool inBasic = false; + bool inLiteral = false; + for (var i = 0; i < line.Length; i++) + { + var c = line[i]; + if (c == '"' && !inLiteral) + { + inBasic = !inBasic; + } + else if (c == '\'' && !inBasic) + { + inLiteral = !inLiteral; + } + else if (c == '#' && !inBasic && !inLiteral) + { + return line.Substring(0, i); + } + } + + return line; + } +} diff --git a/src/Copybara.Core/Credentials/TtlSecret.cs b/src/Copybara.Core/Credentials/TtlSecret.cs new file mode 100644 index 000000000..85d8a1d90 --- /dev/null +++ b/src/Copybara.Core/Credentials/TtlSecret.cs @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2023 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; + +namespace Copybara.Credentials; + +/// A credential with a limited TTL. +public class TtlSecret : StaticSecret +{ + private readonly DateTimeOffset _ttl; + private readonly Func _clock; + + public TtlSecret(string secret, string name, DateTimeOffset ttl, Func clock) + : base(name, secret) + { + _ttl = ttl; + _clock = Preconditions.CheckNotNull(clock); + } + + public override string PrintableValue() => + $""; + + public override string ProvideSecret() + { + DateTimeOffset now = _clock(); + if (_ttl < now) + { + throw new CredentialRetrievalException( + string.Format( + "Credential {0} expired {1} seconds ago.", + PrintableValue(), (long)(now - _ttl).TotalSeconds)); + } + + return base.ProvideSecret(); + } + + public override bool Valid() => _ttl > _clock().AddSeconds(/* 10s grace */ 10); + + public override string ToString() => PrintableValue(); +} diff --git a/src/Copybara.Core/Credentials/UsernamePasswordIssuer.cs b/src/Copybara.Core/Credentials/UsernamePasswordIssuer.cs new file mode 100644 index 000000000..8e65760e6 --- /dev/null +++ b/src/Copybara.Core/Credentials/UsernamePasswordIssuer.cs @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2023 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Starlark.Eval; + +namespace Copybara.Credentials; + +/// A username/password pair issuer. +/// +/// In upstream this is a nested @AutoValue class of CredentialModule. It is lifted to +/// a top-level record here so peer packages can reference it as +/// Copybara.Credentials.UsernamePasswordIssuer. +/// +public sealed record UsernamePasswordIssuer( + CredentialIssuer Username, + CredentialIssuer Password) : IStarlarkValue +{ + public ImmutableArray> DescribeCredentials() => + ImmutableArray.Create(Username.Describe(), Password.Describe()); + + public static UsernamePasswordIssuer Create( + CredentialIssuer username, CredentialIssuer password) => new(username, password); +} diff --git a/src/Copybara.Core/Destination.cs b/src/Copybara.Core/Destination.cs new file mode 100644 index 000000000..aa5e0c8b6 --- /dev/null +++ b/src/Copybara.Core/Destination.cs @@ -0,0 +1,173 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Security.Cryptography; +using Copybara.Common; +using Copybara.Effect; +using Copybara.Revision; +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; +using Console = Copybara.Util.Console.Console; + +namespace Copybara; + +/// A repository which a source of truth can be copied to. +/// the revision type this destination handles. +[StarlarkBuiltin("destination", Doc = "A repository which a source of truth can be copied to")] +public interface IDestination : IConfigItemDescription, IStarlarkValue + where R : class, IRevision +{ + /// + /// Creates a writer which is capable of writing to this destination. This writer may maintain + /// state between writing of revisions. + /// + /// + /// if the writer could not be created because of a user error. + /// + IWriter NewWriter(WriterContext writerContext); + + /// + /// Given a reverse workflow with an Origin that is of the same type as this destination, + /// the label that IOrigin.GetLabelName() would return. + /// + string GetLabelNameWhenOrigin(); + + /// + /// A hash function that is preferred by the Destination for use cases where hashing is involved. + /// + HashAlgorithmName GetHashFunction() => HashAlgorithmName.SHA256; + + /// + /// An object which is capable of writing multiple revisions to the destination. This object is + /// allowed to maintain state between the writing of revisions if applicable. + /// + /// the revision type. + interface IWriter : IChangeVisitable + where TR : class, IRevision + { + /// Returns the status of the import at the destination. + DestinationStatus? GetDestinationStatus(Glob destinationFiles, string labelName); + + /// + /// Returns true if this destination stores revisions in the repository so that + /// can be used for discovering the state of the + /// destination and we can use the methods in . + /// + bool SupportsHistory(); + + /// Writes the fully-transformed repository stored at workdir to this destination. + IReadOnlyList Write( + TransformResult transformResult, Glob destinationFiles, Console console); + + /// Utility endpoint for accessing and adding feedback data. + IEndpoint GetFeedbackEndPoint(Console console) => IEndpoint.NoopEndpoint; + + DestinationReader GetDestinationReader( + Console console, Origin.Baseline? baseline, string workdir) => + DestinationReader.NotImplemented; + + DestinationReader GetDestinationReader(Console console, string? baseline, string workdir) => + DestinationReader.NotImplemented; + + IPatchRegenerator? GetPatchRegenerator(Console console) => null; + + /// Returns the object for this destination. + IDestinationInfo? GetDestinationInfo() => null; + } + + /// Writers that implement PatchRegenerator can be used with RegenerateCmd. + interface IPatchRegenerator + { + /// + /// Write the files in the workdir to an already-existing change created by Copybara. This is + /// used to update a pending change with new patch files. + /// + void UpdateChange( + string workflowName, string workdir, Glob destinationFiles, string changeToUpdate) => + throw new Exceptions.ValidationException( + "update change not implemented for this destination"); + + /// Detect regen baseline when not supplied by CLI. + string? InferRegenBaseline() => null; + + /// Detect regen target when not supplied by CLI. + string? InferRegenTarget() => null; + + /// Detect import baseline when not supplied by CLI. + string? InferImportBaseline(string regenTarget, string workdir) => null; + } +} + +/// +/// This class represents the status of the destination. It includes the baseline revision and, if +/// it is a code review destination, the list of pending changes that have been already migrated. +/// In order: first change is the oldest one. +/// +/// Port of the nested Destination.DestinationStatus Java type (kept at top level here +/// as it does not depend on the destination's revision type). +/// +public sealed class DestinationStatus +{ + private readonly string _baseline; + private readonly ImmutableArray _pendingChanges; + + public DestinationStatus(string baseline, IReadOnlyList pendingChanges) + { + _baseline = Preconditions.CheckNotNull(baseline); + _pendingChanges = Preconditions.CheckNotNull(pendingChanges).ToImmutableArray(); + } + + /// String representation of the latest migrated revision in the baseline. + public string GetBaseline() => + Preconditions.CheckNotNull(_baseline, "Trying to get baseline for NO_STATUS"); + + /// + /// String representation of the migrated revisions that are in pending state in the destination. + /// First element is the oldest one. Last element the newest one. + /// + public IReadOnlyList GetPendingChanges() => _pendingChanges; + + public override bool Equals(object? o) + { + if (ReferenceEquals(this, o)) + { + return true; + } + if (o is null || GetType() != o.GetType()) + { + return false; + } + var that = (DestinationStatus)o; + return string.Equals(_baseline, that._baseline) + && _pendingChanges.SequenceEqual(that._pendingChanges); + } + + public override int GetHashCode() + { + var hash = new HashCode(); + hash.Add(_baseline); + foreach (var c in _pendingChanges) + { + hash.Add(c); + } + return hash.ToHashCode(); + } + + public override string ToString() => + $"DestinationStatus{{baseline={_baseline}, pendingChanges=[{string.Join(", ", _pendingChanges)}]}}"; +} diff --git a/src/Copybara.Core/DestinationInfo.cs b/src/Copybara.Core/DestinationInfo.cs new file mode 100644 index 000000000..f3b903981 --- /dev/null +++ b/src/Copybara.Core/DestinationInfo.cs @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara; + +/// +/// An object that is used by destinations to store data that consists of complex values. +/// +/// Normally, values could be passed to destinations using labels, but the values can only be +/// strings. +/// +public interface IDestinationInfo +{ +} diff --git a/src/Copybara.Core/DestinationReader.cs b/src/Copybara.Core/DestinationReader.cs new file mode 100644 index 000000000..1f52f57fe --- /dev/null +++ b/src/Copybara.Core/DestinationReader.cs @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Exceptions; +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara; + +/// An api handle to read files from the destination, rather than just the origin. +[StarlarkBuiltin("destination_reader", Doc = "Handle to read from the destination")] +public abstract class DestinationReader : IStarlarkValue +{ + public static readonly DestinationReader NotImplemented = new NotImplementedReader(); + + public static readonly DestinationReader NoopDestinationReader = new NoopReader(); + + [StarlarkMethod("read_file", Doc = "Read a file from the destination.")] + public abstract string ReadFile( + [Param(Name = "path", Named = true, Doc = "Path to the file.")] string path); + + [StarlarkMethod( + "copy_destination_files", + Doc = "Copy files from the destination into the workdir.")] + // TODO(joshgoldman): refactor this out in favor of directory-specific version + public abstract void CopyDestinationFiles( + [Param( + Name = "glob", + Named = true, + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) }, + Doc = "Files to copy to the workdir, potentially overwriting files checked out from the" + + " origin.")] + object glob, + [Param( + Name = "path", + Named = true, + Doc = "Optional path to copy the files to", + AllowedTypes = new[] { typeof(CheckoutPath), typeof(NoneType) }, + DefaultValue = "None")] + object path); + + /// + /// Similar to but specifies a destination directory (instead + /// of using the default working directory workdir). + /// + public abstract void CopyDestinationFilesToDirectory(Glob glob, string directory); + + [StarlarkMethod( + "file_exists", + Doc = "Checks whether a given file exists in the destination.")] + public abstract bool Exists( + [Param(Name = "path", Named = true, Doc = "Path to the file.")] string path); + + /// Fetch the destination version at which this file was last modified. + public virtual string? LastModified(string path) => + throw new NotSupportedException( + "Last modified is not implemented in this destination reader."); + + /// + /// Returns true if this implementation supports . + /// + /// If this returns false, hashes will be computed by reading the files from the local + /// filesystem instead. An implementation should only provide getHash() if it can compute the + /// hashes in a more efficient way. + /// + public virtual bool SupportsGetHash() => false; + + /// Obtain the hash of the destination file at this path. + public virtual string GetHash(string path) => + throw new NotSupportedException("Get hash is not implemented in this destination reader."); + + private sealed class NotImplementedReader : DestinationReader + { + public override string ReadFile(string path) => + throw new RepoException("Reading files is not implemented by this destination"); + + public override void CopyDestinationFiles(object glob, object path) => + throw new RepoException("Reading files is not implemented by this destination"); + + public override void CopyDestinationFilesToDirectory(Glob glob, string directory) => + throw new RepoException("Reading files is not implemented by this destination"); + + public override bool Exists(string path) => false; + } + + private sealed class NoopReader : DestinationReader + { + public override string ReadFile(string path) => ""; + + public override void CopyDestinationFiles(object glob, object path) { } + + public override void CopyDestinationFilesToDirectory(Glob glob, string directory) { } + + public override bool Exists(string path) => false; + } +} diff --git a/src/Copybara.Core/DestinationStatusVisitor.cs b/src/Copybara.Core/DestinationStatusVisitor.cs new file mode 100644 index 000000000..323b63057 --- /dev/null +++ b/src/Copybara.Core/DestinationStatusVisitor.cs @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Revision; +using Copybara.Util; + +namespace Copybara; + +/// +/// A visitor that computes the matching the actual files affected by +/// the changes with the destination files glob. +/// +public class DestinationStatusVisitor : IChangesVisitor +{ + private readonly IPathMatcher _pathMatcher; + private readonly string _labelName; + + private DestinationStatus? _destinationStatus; + + public DestinationStatusVisitor(IPathMatcher pathMatcher, string labelName) + { + _pathMatcher = pathMatcher; + _labelName = labelName; + } + + public VisitResult Visit(Change change) + { + var changeFiles = change.GetChangeFiles(); + if (changeFiles != null) + { + if (change.GetLabels().ContainsKey(_labelName)) + { + foreach (var file in changeFiles) + { + if (_pathMatcher.Matches("/" + file)) + { + var values = change.GetLabels().Get(_labelName); + string lastRev = values[values.Length - 1]; + _destinationStatus = + new DestinationStatus(lastRev, ImmutableArray.Empty); + return VisitResult.Terminate; + } + } + } + } + return VisitResult.Continue; + } + + public DestinationStatus? GetDestinationStatus() => _destinationStatus; +} diff --git a/src/Copybara.Core/Doc/AnnotationProcessor.cs b/src/Copybara.Core/Doc/AnnotationProcessor.cs new file mode 100644 index 000000000..7003969a5 --- /dev/null +++ b/src/Copybara.Core/Doc/AnnotationProcessor.cs @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2021 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Doc; + +// Port note: com.google.copybara.doc.AnnotationProcessor is a build-time Java annotation processor +// (extends com.google.auto.common.BasicAnnotationProcessor). At compile time it scanned each jar for +// @StarlarkBuiltin/@Library types and emitted a "starlark_class_list.txt" resource inside the jar, +// which the Generator/ModuleLoader later read back to know which classes to document. +// +// The .NET port has no equivalent build-time processor and no proto/class-list pipeline. Its role is +// fully subsumed by runtime reflection in ModuleLoader (see the "Port note" there), which discovers +// [StarlarkBuiltin]/[Library]-annotated types directly from the loaded assemblies. There is +// therefore no type to port here. +// +// TODO(port): if a design ever needs the precomputed class list (e.g. for AOT/trimming scenarios +// where reflection over all assemblies is undesirable), reintroduce a source generator that emits an +// embedded resource analogous to starlark_class_list.txt. diff --git a/src/Copybara.Core/Doc/Annotations/DocAnnotations.cs b/src/Copybara.Core/Doc/Annotations/DocAnnotations.cs new file mode 100644 index 000000000..22dd1a68a --- /dev/null +++ b/src/Copybara.Core/Doc/Annotations/DocAnnotations.cs @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Doc.Annotations; + +/// +/// Documentation for elements of Copybara configuration, like Origins, Destinations, etc. Port of +/// com.google.copybara.doc.annotations.DocElement. +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, Inherited = false)] +public sealed class DocElementAttribute : Attribute +{ + /// Text explaining what the element does and how to use it. + public string Description { get; set; } = ""; + + /// Kind of the element, can be Origin, Destination, etc. + public Type ElementKind { get; set; } = typeof(object); + + /// Associated flag classes annotated with . + public Type[] Flags { get; set; } = Array.Empty(); +} + +/// +/// A field documentation for a type (in practice, used on enum +/// members). Port of com.google.copybara.doc.annotations.DocField. +/// +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Field | AttributeTargets.Property)] +public sealed class DocFieldAttribute : Attribute +{ + public DocFieldAttribute(string description) => Description = description; + + public string Description { get; } + + public bool Required { get; set; } = true; + + public string DefaultValue { get; set; } = "none"; + + public bool Undocumented { get; set; } + + public bool Deprecated { get; set; } + + /// + /// Use when the elements of a list field are always of the same type so that we can avoid + /// using !FieldClass. + /// + public Type ListType { get; set; } = typeof(object); +} + +/// +/// Annotation for documenting fields for a [Param] or return types. Repeatable in the .NET +/// port via . Port of +/// com.google.copybara.doc.annotations.DocDefault (its container +/// DocDefaults is folded into the repeatable usage here). +/// +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, AllowMultiple = true)] +public sealed class DocDefaultAttribute : Attribute +{ + public DocDefaultAttribute(string field) => Field = field; + + public string Field { get; } + + /// The documented default value (Java's confusingly named value()). + public string Value { get; set; } = ""; + + public string[] AllowedTypes { get; set; } = Array.Empty(); +} + +/// +/// Adds a custom prefix to the signature example and reference in the generated Markdown. Port of +/// com.google.copybara.doc.annotations.DocSignaturePrefix. +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, Inherited = false)] +public sealed class DocSignaturePrefixAttribute : Attribute +{ + public DocSignaturePrefixAttribute(string value) => Value = value; + + /// + /// When generating documentation, use varPrefix + "." + method/field. For example + /// ctx.origin. + /// + public string Value { get; } +} + +/// +/// Associates an example snippet with a configuration element. Repeatable in the .NET port. Port of +/// com.google.copybara.doc.annotations.Example (its container Examples is folded into +/// the repeatable usage here). +/// +[AttributeUsage( + AttributeTargets.Class | AttributeTargets.Interface | AttributeTargets.Field + | AttributeTargets.Method | AttributeTargets.Property, + AllowMultiple = true)] +public sealed class ExampleAttribute : Attribute +{ + public ExampleAttribute(string title, string before, string code) + { + Title = title; + Before = before; + Code = code; + } + + /// Title of the example. + public string Title { get; } + + /// Description shown before the snippet. + public string Before { get; } + + /// The code of the snippet, e.g. core.move('', 'foo'). + public string Code { get; } + + /// Description shown after the code snippet. + public string After { get; set; } = ""; + + /// + /// If set, the test should check for an existing variable in . Otherwise it is + /// assumed to be an expression. + /// + public string TestExistingVariable { get; set; } = ""; +} + +/// +/// Marks a class whose [StarlarkMethod]-annotated methods are predeclared in the environment +/// and added to the generated documentation. Port of +/// com.google.copybara.doc.annotations.Library. +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, Inherited = false)] +public sealed class LibraryAttribute : Attribute +{ +} + +/// +/// Associates flags with functions in Starlark modules. Can be set on a whole module or on specific +/// methods. Port of com.google.copybara.doc.annotations.UsesFlags. +/// +[AttributeUsage( + AttributeTargets.Class | AttributeTargets.Interface | AttributeTargets.Field + | AttributeTargets.Method | AttributeTargets.Property)] +public sealed class UsesFlagsAttribute : Attribute +{ + public UsesFlagsAttribute(params Type[] value) => Value = value; + + /// Associated flag classes annotated with . + public Type[] Value { get; } +} diff --git a/src/Copybara.Core/Doc/DocBase.cs b/src/Copybara.Core/Doc/DocBase.cs new file mode 100644 index 000000000..4f49ded58 --- /dev/null +++ b/src/Copybara.Core/Doc/DocBase.cs @@ -0,0 +1,197 @@ +/* + * Copyright (C) 2021 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Doc.Annotations; + +namespace Copybara.Doc; + +/// +/// Helper for generating documentation from the Starlark annotations in the Copybara codebase. Port +/// of com.google.copybara.doc.DocBase. +/// +public abstract class DocBase : IComparable +{ + protected DocBase(string name, string description, bool isDocumented) + { + Name = name ?? throw new ArgumentNullException(nameof(name)); + Description = description ?? throw new ArgumentNullException(nameof(description)); + IsDocumented = isDocumented; + } + + public string Name { get; } + + public string Description { get; } + + public bool IsDocumented { get; } + + public int CompareTo(DocBase? other) + { + if (other is null) + { + return 1; + } + return string.CompareOrdinal(Name, other.Name); + } + + private static string HandleType(string? type) => type ?? "NoneType"; + + /// Module level documentation node. + public sealed class DocModule : DocBase + { + public DocModule(string name, string description, bool isDocumented) + : base(name, description, isDocumented) + { + } + + // TreeSet in Java -> SortedSet with the natural (name-based) ordering of DocBase. + public SortedSet Fields { get; } = new(DocBaseComparer.Instance); + + public SortedSet Functions { get; } = new(DocBaseComparer.Instance); + + public SortedSet Flags { get; } = new(DocBaseComparer.Instance); + + public override string ToString() => $"DocModule{{name={Name}}}"; + + public ImmutableHashSet GetFunctions() => Functions.ToImmutableHashSet(); + + public ImmutableHashSet GetFields() => Fields.ToImmutableHashSet(); + } + + /// Command line flag documentation node. + public sealed class DocFlag : DocBase + { + public DocFlag(string name, string type, string description, bool isDocumented) + : base(name, description, isDocumented) + { + Type = type; + } + + public string Type { get; } + } + + /// Function level documentation node. + public sealed class DocFunction : DocBase + { + public DocFunction( + string name, + string description, + string? returnType, + IEnumerable parameters, + IEnumerable flags, + IEnumerable examples, + bool hasStar, + bool hasStarStar, + bool isSelfCall, + bool isDocumented) + : base(name, description, isDocumented) + { + ReturnType = returnType; + Params = parameters.ToImmutableArray(); + Examples = examples.ToImmutableArray(); + HasStar = hasStar; + HasStarStar = hasStarStar; + IsSelfCall = isSelfCall; + foreach (DocFlag flag in flags) + { + Flags.Add(flag); + } + } + + public SortedSet Flags { get; } = new(DocBaseComparer.Instance); + + public string? ReturnType { get; } + + public ImmutableArray Params { get; } + + public ImmutableArray Examples { get; } + + public bool HasStar { get; } + + public bool HasStarStar { get; } + + public bool IsSelfCall { get; } + + public IReadOnlyList GetParams() => Params; + + public string GetReturnType() => HandleType(ReturnType); + } + + /// Function parameter level documentation node. + public sealed class DocParam : DocBase + { + public DocParam( + string name, + string? defaultValue, + IReadOnlyList allowedTypes, + string description, + bool isDocumented) + : base(name, description, isDocumented) + { + DefaultValue = defaultValue; + AllowedTypes = allowedTypes; + } + + public string? DefaultValue { get; } + + public IReadOnlyList AllowedTypes { get; } + + public IReadOnlyList GetAllowedTypes() => AllowedTypes; + } + + /// Wrapper around an for a documented element. + public sealed class DocExample + { + public DocExample(ExampleAttribute example) => Example = example; + + public ExampleAttribute Example { get; } + } + + /// Field level documentation node. + public sealed class DocField : DocBase + { + public DocField(string name, string description, string? type, bool isDocumented) + : base(name, description, isDocumented) + { + Type = type; + } + + /// Raw (possibly null) Starlark type name. + public string? Type { get; } + + /// Type name with null normalized to NoneType (Java's getType()). + public string GetResolvedType() => HandleType(Type); + } + + /// Comparer that orders instances by name (Java natural order). + internal sealed class DocBaseComparer : IComparer + { + internal static readonly DocBaseComparer Instance = new(); + + public int Compare(DocBase? x, DocBase? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + if (x is null) + { + return -1; + } + return x.CompareTo(y); + } + } +} diff --git a/src/Copybara.Core/Doc/Generator.cs b/src/Copybara.Core/Doc/Generator.cs new file mode 100644 index 000000000..a2dd2b7ee --- /dev/null +++ b/src/Copybara.Core/Doc/Generator.cs @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Reflection; +using static Copybara.Doc.DocBase; + +namespace Copybara.Doc; + +/// +/// Generates a Markdown document with the Copybara reference guide. Port of +/// com.google.copybara.doc.Generator. +/// +/// Port note: Java's Generator.main took a comma-separated list of jar paths, +/// read a starlark_class_list.txt out of each jar (emitted by the build-time +/// AnnotationProcessor), loaded those classes and reflected over them. The .NET port has no +/// annotation processor, so this generator reflects directly over the module types found in the +/// supplied assemblies (see ). The template-substitution behavior is +/// preserved. +/// +public static class Generator +{ + private const string TemplateReplacement = ""; + + /// + /// Renders the reference Markdown for the given module and returns + /// it, optionally substituted into . + /// + public static string Generate( + IEnumerable assemblies, + IEnumerable? additionalTypes = null, + string? template = null, + bool includeFlagAggregate = true) + { + ImmutableArray modules = new ModuleLoader().Load(assemblies, additionalTypes); + return Render(modules, template, includeFlagAggregate); + } + + /// + /// Renders the reference Markdown from an explicit list of module types. Useful for tests and for + /// callers that already resolved the type list. + /// + public static string GenerateFromTypes( + IEnumerable types, string? template = null, bool includeFlagAggregate = true) + { + ImmutableArray modules = new ModuleLoader().LoadTypes(types); + return Render(modules, template, includeFlagAggregate); + } + + private static string Render( + ImmutableArray modules, string? template, bool includeFlagAggregate) + { + string markdown = new MarkdownRenderer().Render(modules, includeFlagAggregate); + string effectiveTemplate = template ?? TemplateReplacement; + return effectiveTemplate.Replace( + TemplateReplacement, TemplateReplacement + "\n" + markdown); + } + + /// + /// Writes the reference Markdown for the given assemblies to , + /// substituting into when provided. Mirrors Java's + /// Generator.main file-writing behavior. + /// + public static void Write( + IEnumerable assemblies, + string outputFile, + IEnumerable? additionalTypes = null, + string? templateFile = null, + bool includeFlagAggregate = true) + { + string? template = templateFile != null ? File.ReadAllText(templateFile) : null; + string output = Generate(assemblies, additionalTypes, template, includeFlagAggregate); + File.WriteAllText(outputFile, output + Environment.NewLine); + } +} diff --git a/src/Copybara.Core/Doc/MarkdownRenderer.cs b/src/Copybara.Core/Doc/MarkdownRenderer.cs new file mode 100644 index 000000000..d7d9b8520 --- /dev/null +++ b/src/Copybara.Core/Doc/MarkdownRenderer.cs @@ -0,0 +1,376 @@ +/* + * Copyright (C) 2021 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text; +using System.Text.RegularExpressions; +using Copybara.Doc.Annotations; +using static Copybara.Doc.DocBase; + +namespace Copybara.Doc; + +/// +/// Renders a collection of nodes into the Copybara reference Markdown. Port +/// of com.google.copybara.doc.MarkdownRenderer. +/// +internal sealed class MarkdownRenderer +{ + private const int ModuleHeadingLevel = 2; + + private const string SequenceOfPrefix = "sequence of "; + + private readonly HashSet headings = new(); + + private readonly Dictionary> returnedBy = new(); + + private readonly Dictionary> consumedBy = new(); + + private static void AddToMapValueSet( + Dictionary> map, string key, string value) + { + ImmutableHashSet existing = + map.TryGetValue(key, out ImmutableHashSet? set) ? set : ImmutableHashSet.Empty; + map[key] = existing.Add(value); + } + + public string Render(IEnumerable modules, bool includeFlagAggregate) + { + var materialized = modules.ToList(); + var modulesToRender = new List(); + modulesToRender.AddRange(materialized.Where(m => m.IsDocumented)); + if (includeFlagAggregate) + { + modulesToRender.Add(RenderFlags(materialized)); + } + + PopulateUsageMaps(modulesToRender); + + var sb = new StringBuilder(); + sb.Append(TableOfContents(modulesToRender)); + + foreach (DocModule module in modulesToRender) + { + sb.Append('\n'); + sb.Append(RenderModule(module, ModuleHeadingLevel)); + } + return sb.ToString(); + } + + private void PopulateUsageMaps(IEnumerable modules) + { + foreach (DocModule module in modules.Where(m => m.IsDocumented)) + { + foreach (DocFunction f in module.Functions.Where(x => x.IsDocumented)) + { + if (f.ReturnType != null) + { + AddToMapValueSet(returnedBy, f.ReturnType, f.Name); + + if (f.ReturnType.StartsWith(SequenceOfPrefix, StringComparison.Ordinal)) + { + AddToMapValueSet(returnedBy, GetSequenceElementType(f.ReturnType), f.Name); + } + if (f.ReturnType.StartsWith("dict[", StringComparison.Ordinal)) + { + AddToMapValueSet(returnedBy, GetDictKeyType(f.ReturnType), f.Name); + AddToMapValueSet(returnedBy, GetDictValueType(f.ReturnType), f.Name); + } + } + + foreach (DocParam param in f.Params.Where(x => x.IsDocumented)) + { + foreach (string type in param.AllowedTypes) + { + AddToMapValueSet(consumedBy, type, f.Name); + + if (type.StartsWith(SequenceOfPrefix, StringComparison.Ordinal)) + { + AddToMapValueSet(consumedBy, GetSequenceElementType(type), f.Name); + } + if (type.StartsWith("dict[", StringComparison.Ordinal)) + { + AddToMapValueSet(consumedBy, GetDictKeyType(type), f.Name); + AddToMapValueSet(consumedBy, GetDictValueType(type), f.Name); + } + } + } + } + } + } + + private DocModule RenderFlags(IEnumerable modules) + { + var flagSet = new SortedSet(DocBaseComparer.Instance); + foreach (DocModule module in modules.Where(m => m.IsDocumented)) + { + foreach (DocFlag flag in module.Flags) + { + flagSet.Add(flag); + } + } + var flagModule = + new DocModule("copybara_flags", "All flag options available to the Copybara CLI.", true); + foreach (DocFlag flag in flagSet) + { + flagModule.Flags.Add(flag); + } + return flagModule; + } + + private string TableOfContents(IEnumerable modules) + { + var sb = new StringBuilder(); + sb.Append("## Table of Contents\n\n\n"); + foreach (DocModule module in modules) + { + headings.Add(module.Name); + sb.Append(" - "); + sb.Append(Linkify(module.Name)); + sb.Append('\n'); + foreach (DocFunction f in module.Functions.Where(x => x.IsDocumented)) + { + headings.Add(f.Name); + sb.Append(" - "); + sb.Append(Linkify(f.Name)); + sb.Append('\n'); + } + } + sb.Append('\n'); + return sb.ToString(); + } + + private string RenderModule(DocModule module, int level) + { + var sb = new StringBuilder(); + sb.Append(Title(level, module.Name)); + sb.Append(module.Description).Append("\n\n"); + + if (module.Fields.Count > 0) + { + sb.Append(HtmlTitle(level + 2, "Fields:", "fields." + module.Name)); + sb.Append(TableHeader("Name", "Description")); + foreach (DocField field in module.Fields) + { + sb.Append( + TableRow( + field.Name, + $"{TypeName(field.GetResolvedType())}

{field.Description}

")); + } + sb.Append('\n'); + } + + sb.Append(RenderFlagsTable(module.Flags)); + + ImmutableHashSet moduleReturnedBy = GetOrEmpty(returnedBy, module.Name); + if (!moduleReturnedBy.IsEmpty) + { + sb.Append(HtmlTitle(level + 2, "Returned By:", "returned_by." + module.Name)); + sb.Append("
    "); + foreach (string funcName in moduleReturnedBy) + { + sb.Append($"
  • {funcName}
    • "); + } + sb.Append("
"); + } + ImmutableHashSet moduleConsumedBy = GetOrEmpty(consumedBy, module.Name); + if (!moduleConsumedBy.IsEmpty) + { + sb.Append(HtmlTitle(level + 2, "Consumed By:", "consumed_by." + module.Name)); + sb.Append("
    "); + foreach (string funcName in moduleConsumedBy) + { + sb.Append($"
  • {funcName}
    • "); + } + sb.Append("
"); + } + + if (!moduleReturnedBy.IsEmpty || !moduleConsumedBy.IsEmpty) + { + sb.Append("\n\n"); + } + + foreach (DocFunction func in module.Functions.Where(x => x.IsDocumented)) + { + sb.Append(""); + sb.Append(Title(level + 1, func.Name)); + sb.Append(func.Description); + sb.Append("\n\n"); + if (func.ReturnType != null) + { + sb.Append(TypeName(func.ReturnType)).Append(' '); + } + sb.Append("").Append(func.Name).Append('('); + sb.Append( + string.Join( + ", ", + func.Params.Select( + p => + $"{p.Name}" + + (p.DefaultValue == null ? "" : "=" + p.DefaultValue)))); + sb.Append(")\n\n"); + + if (func.Params.Length > 0) + { + sb.Append(HtmlTitle(level + 2, "Parameters:", $"parameters.{func.Name}")); + sb.Append(TableHeader("Parameter", "Description")); + foreach (DocParam param in func.Params.Where(x => x.IsDocumented)) + { + sb.Append( + TableRow( + $"{param.Name}", + $"{string.Join(" or ", param.AllowedTypes.Select(TypeName))}

{param.Description}

")); + } + sb.Append('\n'); + } + if (func.Examples.Length > 0) + { + sb.Append( + HtmlTitle( + level + 2, + func.Examples.Length == 1 ? "Example:" : "Examples:", + "example." + func.Name)); + foreach (DocExample example in func.Examples) + { + sb.Append(RenderExample(level + 3, example.Example)); + } + sb.Append('\n'); + } + sb.Append(RenderFlagsTable(func.Flags)); + } + return FixUpBazelDoc(sb); + } + + // Bazel has some html that assumes a different context, hacky best effort correction. + private static string FixUpBazelDoc(StringBuilder doc) + { + string bazelDoc = doc.ToString(); + bazelDoc = bazelDoc.Replace("../core/set.html", "#set-2"); + bazelDoc = bazelDoc.Replace("../globals/all.html", ""); + bazelDoc = Regex.Replace( + bazelDoc, + @"(?|
    |
      ))\s*(
    • |
|)", + "$1", + RegexOptions.Singleline | RegexOptions.Multiline); + bazelDoc = Regex.Replace( + bazelDoc, @"(\s*)", "$1", RegexOptions.Singleline | RegexOptions.Multiline); + bazelDoc = Regex.Replace( + bazelDoc, @"(<[ou]l>)(\s*)", "$1$2", RegexOptions.Singleline | RegexOptions.Multiline); + return bazelDoc; + } + + private static string Title(int level, string name) => "\n" + new string('#', level) + ' ' + name + "\n\n"; + + private static string HtmlTitle(int level, string name, string id) + { + string tag = $"h{level}"; + return $"\n<{tag} id=\"{id}\">{name}\n\n"; + } + + private bool ShouldLinkify(string type) => headings.Contains(type); + + private string TypeName(string type) => HtmlCodify(TypeNameHelper(type)); + + private static string GetSequenceElementType(string sequenceType) => + sequenceType.Substring(SequenceOfPrefix.Length); + + private static string GetDictKeyType(string dictType) => + dictType.Substring("dict[".Length, dictType.IndexOf(", ", StringComparison.Ordinal) - "dict[".Length); + + private static string GetDictValueType(string dictType) => + dictType.Substring( + dictType.IndexOf(", ", StringComparison.Ordinal) + 2, + dictType.IndexOf(']') - (dictType.IndexOf(", ", StringComparison.Ordinal) + 2)); + + // type name without 'code' formatting applied + private string TypeNameHelper(string type) + { + if (type.StartsWith(SequenceOfPrefix, StringComparison.Ordinal)) + { + return SequenceOfPrefix + TypeNameHelper(GetSequenceElementType(type)); + } + + if (type.StartsWith("dict[", StringComparison.Ordinal)) + { + return "dict[" + + TypeNameHelper(GetDictKeyType(type)) + + ", " + + TypeNameHelper(GetDictValueType(type)) + + "]"; + } + + if (ShouldLinkify(type)) + { + // use html tags, not markdown links, for correct nesting behavior + return HtmlLinkify(type); + } + return type; + } + + private static string Linkify(string name) => + "[" + name + "](#" + name.ToLowerInvariant().Replace(".", "").Replace("`", "") + ")"; + + private static string HtmlLinkify(string name) + { + string href = "#" + name.ToLowerInvariant().Replace(".", "").Replace("`", ""); + return $"{name}"; + } + + private static string HtmlCodify(string snippet) => $"{snippet}"; + + private static string RenderExample(int level, ExampleAttribute example) + { + var sb = new StringBuilder(); + sb.Append(Title(level, example.Title + ":")); + sb.Append(example.Before).Append("\n\n"); + sb.Append("```python\n").Append(example.Code).Append("\n```\n\n"); + if (example.After.Length != 0) + { + sb.Append(example.After).Append("\n\n"); + } + return sb.ToString(); + } + + private static string RenderFlagsTable(IEnumerable flags) + { + var list = flags.ToList(); + var sb = new StringBuilder(); + if (list.Count > 0) + { + sb.Append("\n\n**Command line flags:**\n\n"); + sb.Append(TableHeader("Name", "Type", "Description")); + foreach (DocFlag field in list.Where(f => f.IsDocumented)) + { + sb.Append(TableRow(NoWrap(field.Name), $"*{field.Type}*", field.Description)); + } + sb.Append('\n'); + } + return sb.ToString(); + } + + /// Don't wrap this text. Also use '`' to show it as code. + private static string NoWrap(string text) => + $"`{text}`"; + + private static string TableHeader(params string[] fields) => + TableRow(fields) + TableRow(fields.Select(e => new string('-', e.Length)).ToArray()); + + private static string TableRow(params string[] fields) => + string.Join(" | ", fields.Select(s => s.Replace("\n", "
"))) + "\n"; + + private static ImmutableHashSet GetOrEmpty( + Dictionary> map, string key) => + map.TryGetValue(key, out ImmutableHashSet? set) ? set : ImmutableHashSet.Empty; +} diff --git a/src/Copybara.Core/Doc/ModuleLoader.cs b/src/Copybara.Core/Doc/ModuleLoader.cs new file mode 100644 index 000000000..5922b820c --- /dev/null +++ b/src/Copybara.Core/Doc/ModuleLoader.cs @@ -0,0 +1,407 @@ +/* + * Copyright (C) 2021 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections; +using System.Collections.Immutable; +using System.Net; +using System.Reflection; +using System.Text.RegularExpressions; +using Copybara.Doc.Annotations; +using Starlark.Annot; +using static Copybara.Doc.DocBase; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara.Doc; + +/// +/// Gathers Copybara documentation by reflecting over the loaded module types. Port of +/// com.google.copybara.doc.ModuleLoader. +/// +/// Port note: the Java tool relied on a build-time annotation processor +/// (AnnotationProcessor) that emitted a starlark_class_list.txt file inside each jar, +/// listing the fully-qualified names of all @StarlarkBuiltin/@Library classes; that +/// file was then read out of the jars at doc-generation time. In the .NET port there is no such +/// processor and no proto glue. Instead we discover the module types directly at runtime via +/// — either from an explicit list of s or by +/// scanning the given assemblies for [StarlarkBuiltin]/[Library]-annotated types. +/// This is the pragmatic .NET equivalent of the annotation-processor + class-list-file pipeline. +/// +public sealed class ModuleLoader +{ + /// + /// Loads documentation modules by scanning the given assemblies for annotated types, plus the + /// explicitly supplied . This is the .NET analogue of Java's + /// load(List<String> jarFiles, List<String> additionalClasses): instead of + /// reading a class-list file out of jars, we reflect over loaded assemblies. + /// + public ImmutableArray Load( + IEnumerable assemblies, IEnumerable? additionalTypes = null) + { + var types = new List(); + foreach (Assembly asm in assemblies) + { + foreach (Type t in SafeGetTypes(asm)) + { + if (t.GetCustomAttribute() != null + || t.GetCustomAttribute() != null + || t.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static) + .Any(m => m.GetCustomAttribute() != null)) + { + types.Add(t); + } + } + } + if (additionalTypes != null) + { + types.AddRange(additionalTypes); + } + return LoadTypes(types); + } + + /// + /// Loads documentation from an explicit set of module types (equivalent to Java's resolved class + /// list). Kept public so tests can drive the extractor deterministically. + /// + public ImmutableArray LoadTypes(IEnumerable classes) + { + var modules = new List(); + var docModule = new DocModule("Globals", "Global functions available in Copybara", true); + modules.Add(docModule); + + foreach (Type cls in classes.Distinct()) + { + if (cls.GetCustomAttribute() != null) + { + foreach (DocFunction f in ProcessFunctions(cls, null)) + { + docModule.Functions.Add(f); + } + } + + StarlarkBuiltinAttribute? starlarkBuiltin = cls.GetCustomAttribute(); + if (starlarkBuiltin != null) + { + if (!starlarkBuiltin.Documented) + { + continue; + } + DocSignaturePrefixAttribute? prefixAnn = cls.GetCustomAttribute(); + string prefix = prefixAnn != null ? prefixAnn.Value : starlarkBuiltin.Name; + var mod = new DocModule(starlarkBuiltin.Name, starlarkBuiltin.Doc, starlarkBuiltin.Documented); + foreach (DocFunction f in ProcessFunctions(cls, prefix)) + { + mod.Functions.Add(f); + } + foreach (DocField field in ProcessFields(cls)) + { + mod.Fields.Add(field); + } + foreach (DocFlag flag in GenerateFlagsInfo(cls)) + { + mod.Flags.Add(flag); + } + modules.Add(mod); + continue; + } + + // Globals-only library: any [StarlarkMethod]-annotated method contributes to Globals. + if (GetStarlarkMethods(cls).Any()) + { + foreach (DocFunction f in ProcessFunctions(cls, null)) + { + docModule.Functions.Add(f); + } + } + } + + return DeduplicateAndSort(modules); + } + + private static IEnumerable SafeGetTypes(Assembly asm) + { + try + { + return asm.GetTypes(); + } + catch (ReflectionTypeLoadException e) + { + return e.Types.Where(t => t != null)!.Cast(); + } + } + + private static IEnumerable<(MethodInfo Method, StarlarkMethodAttribute Annotation)> GetStarlarkMethods( + Type cls) + { + foreach (MethodInfo m in cls.GetMethods( + BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static | BindingFlags.FlattenHierarchy)) + { + var ann = m.GetCustomAttribute(); + if (ann != null) + { + yield return (m, ann); + } + } + } + + private IEnumerable ProcessFields(Type cls) + { + return GetStarlarkMethods(cls) + .Where(e => e.Annotation.StructField) + .Select(e => ProcessStarlarkMethod(e.Method, e.Annotation, null)) + .Select(m => new DocField(m.Name, m.Description, m.ReturnType, m.IsDocumented)) + .ToImmutableArray(); + } + + private IEnumerable ProcessFunctions(Type cls, string? prefix) + { + var functions = new List(); + // Java calls Starlark.getSelfCallMethod; here the selfCall method is one annotated with + // SelfCall = true. + foreach (var (method, ann) in GetStarlarkMethods(cls)) + { + if (ann.SelfCall) + { + functions.Add(ProcessStarlarkMethod(method, ann, prefix)); + } + } + functions.AddRange( + GetStarlarkMethods(cls) + .Where(e => !e.Annotation.StructField && !e.Annotation.SelfCall) + .Select(e => ProcessStarlarkMethod(e.Method, e.Annotation, prefix))); + return functions; + } + + private DocFunction ProcessStarlarkMethod( + MethodInfo method, StarlarkMethodAttribute annotation, string? prefix) + { + // In the .NET port, [Param]/[ParamType] annotations live on the C# parameters themselves + // (not in a nested parameters={} array as in Java). Interpreter-supplied parameters + // (StarlarkThread / StarlarkSemantics) are matched by type and skipped here. + ParameterInfo[] clrParams = method.GetParameters(); + var starlarkParams = clrParams + .Where(p => p.GetCustomAttribute() != null) + .ToList(); + + var docDefaultsMap = new Dictionary(); + foreach (DocDefaultAttribute dd in method.GetCustomAttributes()) + { + docDefaultsMap[dd.Field] = dd; + } + + var paramsList = new List(); + foreach (ParameterInfo clrParam in starlarkParams) + { + ParamAttribute starlarkParam = clrParam.GetCustomAttribute()!; + Type parameterType = clrParam.ParameterType; + + // Compute allowed type names (e.g. string or bool or NoneType). + var allowedTypeNames = new List(); + ParamTypeAttribute[] paramTypes = clrParam.GetCustomAttributes().ToArray(); + if (starlarkParam.AllowedTypes.Length > 0) + { + foreach (Type t in starlarkParam.AllowedTypes) + { + allowedTypeNames.Add(SkylarkTypeName(t)); + } + } + else if (paramTypes.Length > 0) + { + foreach (ParamTypeAttribute pt in paramTypes) + { + allowedTypeNames.Add( + SkylarkTypeName(pt.Type) + + (pt.Generic1 != null && pt.Generic1 != typeof(object) + ? " of " + SkylarkTypeName(pt.Generic1) + : "")); + } + } + else + { + allowedTypeNames.Add(SkylarkTypeName(parameterType)); + } + + string paramName = string.IsNullOrEmpty(starlarkParam.Name) ? clrParam.Name ?? "" : starlarkParam.Name; + + docDefaultsMap.TryGetValue(paramName, out DocDefaultAttribute? fieldInfo); + if (fieldInfo != null && fieldInfo.AllowedTypes.Length > 0) + { + allowedTypeNames = fieldInfo.AllowedTypes.ToList(); + } + paramsList.Add( + new DocParam( + paramName, + fieldInfo != null ? fieldInfo.Value : EmptyToNull(starlarkParam.DefaultValue), + allowedTypeNames, + starlarkParam.Doc, + starlarkParam.Documented)); + } + + // Java handles extraKeywords()/extraPositionals() named metadata. The .NET StarlarkMethod + // attribute has no such fields; residual *args/**kwargs are inferred structurally by the + // interpreter (see MethodDescriptor), not documented via metadata here. + // TODO(port): surface extraPositionals/extraKeywords docs if/when the attribute gains them. + bool hasStar = false; + bool hasStarStar = false; + + Type returnType = method.ReturnType; + string? returnTypeName = + returnType == typeof(void) || StarlarkRt.ClassType(returnType) == "NoneType" + ? null + : SkylarkTypeName(returnType); + + string name = prefix != null + ? prefix + (annotation.SelfCall ? "" : "." + annotation.Name) + : annotation.Name; + + var examples = method.GetCustomAttributes().Select(e => new DocExample(e)); + + return new DocFunction( + name, + annotation.Doc, + returnTypeName, + paramsList, + GenerateFlagsInfo(method), + examples, + hasStar, + hasStarStar, + annotation.SelfCall, + annotation.Documented); + } + + private IEnumerable GenerateFlagsInfo(MemberInfo el) + { + var result = new List(); + var usesFlags = el.GetCustomAttribute(); + if (usesFlags == null) + { + return result; + } + foreach (Type c in usesFlags.Value) + { + foreach (MemberInfo m in + c.GetMembers(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static)) + { + if (m is not (PropertyInfo or FieldInfo)) + { + continue; + } + var flag = m.GetCustomAttribute(); + if (flag == null || flag.Hidden) + { + continue; + } + Type memberType = m is PropertyInfo pi ? pi.PropertyType : ((FieldInfo)m).FieldType; + string description = flag.Description; + if (memberType == typeof(TimeSpan)) + { + // Java appended a note for DurationConverter-backed flags. + description += (description.EndsWith(".", StringComparison.Ordinal) ? " " : ". ") + + " Example values: 30s, 20m, 1h, etc."; + } + result.Add( + new DocFlag( + string.Join(", ", flag.Names), + SimplerJavaTypes(memberType), + description, + !flag.Hidden)); + } + } + return result; + } + + private static readonly Regex TypeNameRegex = new("(?:[A-Za-z.]*\\.)*([A-Za-z]+)"); + + private string SimplerJavaTypes(Type s) + { + Type underlying = Nullable.GetUnderlyingType(s) ?? s; + if (underlying.IsEnum) + { + return "`" + string.Join("`
or `", Enum.GetNames(underlying)) + "`"; + } + string result = TypeNameRegex.Replace(underlying.Name, m => DeCapitalize(m.Groups[1].Value)); + return WebUtility.HtmlEncode(result); + } + + private static string DeCapitalize(string substring) => + substring.Length == 0 + ? substring + : char.ToLowerInvariant(substring[0]) + substring.Substring(1); + + /// + /// Best-effort Starlark type name for a CLR type. Simplified relative to Java's generic + /// reflection (which walked ParameterizedType / WildcardType / TypeVariable); .NET generics are + /// handled here for the common dict/sequence collection shapes. + /// + private string SkylarkTypeName(Type type) + { + Type t = Nullable.GetUnderlyingType(type) ?? type; + + if (t.IsGenericType) + { + Type def = t.GetGenericTypeDefinition(); + Type[] args = t.GetGenericArguments(); + + if (typeof(IDictionary).IsAssignableFrom(t) + || def == typeof(IDictionary<,>) + || def == typeof(IReadOnlyDictionary<,>) + || def == typeof(Dictionary<,>)) + { + if (args.Length == 2) + { + return IsObject(args[0]) || IsObject(args[1]) + ? "dict" + : $"dict[{SkylarkTypeName(args[0])}, {SkylarkTypeName(args[1])}]"; + } + } + + if (args.Length == 1 + && typeof(IEnumerable).IsAssignableFrom(t) + && t != typeof(string)) + { + return IsObject(args[0]) ? "sequence" : $"list of {SkylarkTypeName(args[0])}"; + } + + return StarlarkRt.ClassType(t); + } + + if (t.IsGenericParameter) + { + return "?"; + } + + return StarlarkRt.ClassType(t); + } + + private static bool IsObject(Type type) => type == typeof(object); + + private static string? EmptyToNull(string? value) => string.IsNullOrEmpty(value) ? null : value; + + private static ImmutableArray DeduplicateAndSort(IEnumerable modules) + { + var asMap = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + foreach (DocModule module in modules) + { + if (!asMap.TryGetValue(module.Name, out DocModule? existing) + || existing.Functions.Count < module.Functions.Count + || existing.Fields.Count < module.Fields.Count + || existing.Flags.Count < module.Flags.Count) + { + asMap[module.Name] = module; + } + } + return asMap.Values.ToImmutableArray(); + } +} diff --git a/src/Copybara.Core/Effect/DestinationEffect.cs b/src/Copybara.Core/Effect/DestinationEffect.cs new file mode 100644 index 000000000..1deb8717c --- /dev/null +++ b/src/Copybara.Core/Effect/DestinationEffect.cs @@ -0,0 +1,262 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Revision; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Effect; + +/// An effect happening in the destination as a consequence of the migration. +[StarlarkBuiltin( + "destination_effect", + Doc = "Represents an effect that happened in the destination due to a single migration")] +public class DestinationEffect : IStarlarkPrintableValue +{ + private readonly EffectType _type; + private readonly string _summary; + private readonly ImmutableArray _originRefs; + private readonly DestinationRef? _destinationRef; + private readonly ImmutableArray _errors; + + public DestinationEffect( + EffectType type, + string summary, + IEnumerable originRefs, + DestinationRef? destinationRef) + : this(type, summary, originRefs, destinationRef, ImmutableArray.Empty) + { + } + + public DestinationEffect( + EffectType type, + string summary, + IEnumerable originRefs, + DestinationRef? destinationRef, + IEnumerable errors) + { + _type = type; + _summary = Preconditions.CheckNotNull(summary); + _originRefs = Preconditions.CheckNotNull(originRefs).ToImmutableArray(); + _destinationRef = destinationRef; + _errors = Preconditions.CheckNotNull(errors).ToImmutableArray(); + } + + /// Returns the origin references included in this effect. + public IReadOnlyList OriginRefs => _originRefs; + + [StarlarkMethod( + "origin_refs", + Doc = "List of origin changes that were included in this migration", + StructField = true)] + public StarlarkList GetOriginRefsSkylark() => StarlarkList.ImmutableCopyOf(_originRefs); + + /// Return the type of effect that happened: Create, updated, noop or error. + public EffectType Type => _type; + + [StarlarkMethod( + "type", + Doc = + "Return the type of effect that happened: CREATED, UPDATED, NOOP," + + " INSUFFICIENT_APPROVALS or ERROR", + StructField = true)] + public string GetTypeSkylark() => _type.ToString(); + + /// + /// Textual summary of what happened. Users of this class should not try to parse this field. + /// + [StarlarkMethod( + "summary", + Doc = + "Textual summary of what happened. Users of this class should not try to parse this" + + " field.", + StructField = true)] + public string Summary => _summary; + + /// + /// Destination reference updated/created. Might be null if there was no effect. Might be set even + /// if the type is error (for example a synchronous presubmit test failed but a review was + /// created). + /// + [StarlarkMethod( + "destination_ref", + Doc = + "Destination reference updated/created. Might be null if there was no effect. Might be" + + " set even if the type is error (For example a synchronous presubmit test failed" + + " but a review was created).", + StructField = true, + AllowReturnNones = true)] + public DestinationRef? GetDestinationRef() => _destinationRef; + + /// + /// List of errors that happened during the write to the destination. This can be used for + /// example for synchronous presubmit failures. + /// + public IReadOnlyList Errors => _errors; + + [StarlarkMethod( + "errors", + Doc = "List of errors that happened during the migration", + StructField = true)] + public StarlarkList GetErrorsSkylark() => StarlarkList.ImmutableCopyOf(_errors.Cast()); + + public override bool Equals(object? o) + { + if (ReferenceEquals(this, o)) + { + return true; + } + if (o is null || GetType() != o.GetType()) + { + return false; + } + var that = (DestinationEffect)o; + return _type == that._type + && string.Equals(_summary, that._summary) + && _originRefs.SequenceEqual(that._originRefs) + && Equals(_destinationRef, that._destinationRef) + && _errors.SequenceEqual(that._errors); + } + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"DestinationEffect{{type={_type}, summary={_summary}, originRefs=[{string.Join(", ", _originRefs)}]" + + $", destinationRef={_destinationRef?.ToString() ?? "null"}, errors=[{string.Join(", ", _errors)}]}}"; + + public override int GetHashCode() + { + var hash = new HashCode(); + hash.Add(_type); + hash.Add(_summary); + foreach (var r in _originRefs) + { + hash.Add(r); + } + hash.Add(_destinationRef); + foreach (var e in _errors) + { + hash.Add(e); + } + return hash.ToHashCode(); + } + + /// Type of effect on the destination. + public enum EffectType + { + /// A new review or change was created. + CREATED, + + /// An existing review or change was updated. + UPDATED, + + /// The change was a noop, relative to the destination's baseline. + NOOP, + + /// The change was a noop, relative to an existing pending change in the destination. + NOOP_AGAINST_PENDING_CHANGE, + + /// The effect couldn't happen because the change doesn't have enough approvals. + INSUFFICIENT_APPROVALS, + + /// + /// A user attributable error happened that prevented the destination from creating/updating + /// the change. + /// + ERROR, + + /// + /// An error not attributable to the user that could be retried (RepoException, IOException...). + /// + TEMPORARY_ERROR, + + /// + /// A starting effect of a migration that is eventually expected to trigger another migration + /// asynchronously. This allows to have 'dependant' migrations defined by users. An example of + /// this: a workflow migrates code from a Gerrit review to a GitHub PR, and a feedback + /// migration migrates the test results from a CI in GitHub back to the Gerrit change. This + /// effect would be created on the former one. + /// + STARTED, + } + + /// Reference to the change/review created/updated on the destination. + [StarlarkBuiltin( + "destination_ref", + Doc = "Reference to the change/review created/updated on the destination.")] + public class DestinationRef : IStarlarkPrintableValue + { + private readonly string? _url; + private readonly string _id; + private readonly string _type; + + public DestinationRef(string id, string type, string? url) + { + _id = Preconditions.CheckNotNull(id); + _type = Preconditions.CheckNotNull(type); + _url = url; + } + + /// Destination reference id. + [StarlarkMethod("id", Doc = "Destination reference id", StructField = true)] + public string Id => _id; + + /// + /// Type of reference created. Each destination defines its own and guarantees to be more + /// stable than urls/ids. + /// + [StarlarkMethod( + "type", + Doc = + "Type of reference created. Each destination defines its own and guarantees to be" + + " more stable than urls/ids", + StructField = true)] + public string Type => _type; + + /// Url, if any, of the destination change. + [StarlarkMethod( + "url", + Doc = "Url, if any, of the destination change", + StructField = true, + AllowReturnNones = true)] + public string? Url => _url; + + public override bool Equals(object? o) + { + if (ReferenceEquals(this, o)) + { + return true; + } + if (o is null || GetType() != o.GetType()) + { + return false; + } + var that = (DestinationRef)o; + return string.Equals(_url, that._url) + && string.Equals(_id, that._id) + && string.Equals(_type, that._type); + } + + public override int GetHashCode() => HashCode.Combine(_url, _id, _type); + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"DestinationRef{{url={_url ?? "null"}, id={_id}, type={_type}}}"; + } +} diff --git a/src/Copybara.Core/Endpoint.cs b/src/Copybara.Core/Endpoint.cs new file mode 100644 index 000000000..664575e7e --- /dev/null +++ b/src/Copybara.Core/Endpoint.cs @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Copybara.Effect; +using Copybara.Revision; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; +using Console = Copybara.Util.Console.Console; + +namespace Copybara; + +/// +/// An origin or destination API in a feedback migration. +/// +/// Endpoints are symmetric, that is, they need to be able to act both as an origin and +/// destination of a feedback migration, which means that they need to support both read and write +/// operations on the API. +/// +[StarlarkBuiltin("endpoint", Doc = "An origin or destination API in a feedback migration.")] +public interface IEndpoint : IStarlarkPrintableValue, IConfigItemDescription +{ + /// + /// To be used for core.workflow origin/destinations that don't want to provide an api for giving + /// feedback. + /// + public static readonly IEndpoint NoopEndpoint = new NoopEndpointImpl(); + + void IStarlarkPrintableValue.Repr(Printer printer, StarlarkSemantics semantics) => + printer.Append(ToString() ?? ""); + + /// Returns a key-value list of the options the endpoint was instantiated with. + ImmutableListMultimap Describe(); + + [StarlarkMethod( + "new_origin_ref", + Doc = "Creates a new origin reference out of this endpoint.")] + OriginRef NewOriginRef([Param(Name = "ref", Named = true, Doc = "The reference.")] string @ref) => + new(@ref); + + [StarlarkMethod( + "new_destination_ref", + Doc = "Creates a new destination reference out of this endpoint.")] + DestinationEffect.DestinationRef NewDestinationRef( + [Param(Name = "ref", Named = true, Doc = "The reference.")] string @ref, + [Param(Name = "type", Named = true, Doc = "The type of this reference.")] string type, + [Param( + Name = "url", + Named = true, + Doc = "The url associated with this reference, if any.", + DefaultValue = "None")] + object? urlObj) + { + string? url = StarlarkRt.IsNullOrNone(urlObj) ? null : (string?)urlObj; + return new DestinationEffect.DestinationRef(@ref, type, url); + } + + [StarlarkMethod( + "url", + Doc = "Return the URL of this endpoint.", + StructField = true, + AllowReturnNones = true)] + string? GetUrl() => null; + + /// Returns an instance of this endpoint with the given console. + IEndpoint WithConsole(Console console) => this; + + private sealed class NoopEndpointImpl : IEndpoint + { + public ImmutableListMultimap Describe() => + throw new InvalidOperationException("Instance shouldn't be used for core.feedback"); + + void IStarlarkPrintableValue.Repr(Printer printer, StarlarkSemantics semantics) => + printer.Append("noop_endpoint"); + } +} diff --git a/src/Copybara.Core/EndpointProvider.cs b/src/Copybara.Core/EndpointProvider.cs new file mode 100644 index 000000000..72c44d266 --- /dev/null +++ b/src/Copybara.Core/EndpointProvider.cs @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara; + +/// +/// Non-generic view over . Since callers frequently hold an +/// endpoint provider without knowing its concrete endpoint type (Java uses the raw type), this +/// interface exposes the endpoint access that doesn't depend on T. +/// +public interface IEndpointProvider : IStarlarkValue +{ + /// Returns the wrapped endpoint. + IEndpoint GetEndpoint(); + + // TODO(b/269526710): Remove method + ImmutableListMultimap Describe(); +} + +/// Wrapper class to prevent arbitrary instantiation of endpoints in starlark. +[StarlarkBuiltin( + "endpoint_provider", + Doc = "An handle for an origin or destination API in a feedback migration.", + Documented = false)] +public class EndpointProvider : IEndpointProvider + where T : IEndpoint +{ + internal readonly T Endpoint; + + internal EndpointProvider(T endpoint) => Endpoint = endpoint; + + public T GetEndpoint() => Endpoint; + + IEndpoint IEndpointProvider.GetEndpoint() => Endpoint; + + // TODO(b/269526710): Remove method + public ImmutableListMultimap Describe() => Endpoint.Describe(); + + /// Wrap an Endpoint. + public static EndpointProvider Wrap(T e) => new(e); +} + +/// Non-generic factory helpers for . +public static class EndpointProvider +{ + public static EndpointProvider Wrap(T e) + where T : IEndpoint => new(e); +} diff --git a/src/Copybara.Core/Exceptions/AccessValidationException.cs b/src/Copybara.Core/Exceptions/AccessValidationException.cs new file mode 100644 index 000000000..c86c68e81 --- /dev/null +++ b/src/Copybara.Core/Exceptions/AccessValidationException.cs @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Exceptions; + +/// +/// A special case of when the error is likely to be a user error +/// related to access to the repo (access denied, not found, etc.) +/// +public class AccessValidationException : ValidationException +{ + public AccessValidationException(string message) + : base(message) + { + } + + public AccessValidationException(string message, Exception? cause) + : base(message, cause) + { + } +} diff --git a/src/Copybara.Core/Exceptions/CannotResolveLabel.cs b/src/Copybara.Core/Exceptions/CannotResolveLabel.cs new file mode 100644 index 000000000..94909f3fc --- /dev/null +++ b/src/Copybara.Core/Exceptions/CannotResolveLabel.cs @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Exceptions; + +/// +/// An exception thrown when a dependency expressed as a label in a content cannot be resolved. +/// +public class CannotResolveLabel : ValidationException +{ + public CannotResolveLabel(string message) + : base(message) + { + } + + public CannotResolveLabel(string message, Exception? exception) + : base(message, exception) + { + } +} diff --git a/src/Copybara.Core/Exceptions/CannotResolveRevisionException.cs b/src/Copybara.Core/Exceptions/CannotResolveRevisionException.cs new file mode 100644 index 000000000..100e5a66e --- /dev/null +++ b/src/Copybara.Core/Exceptions/CannotResolveRevisionException.cs @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Exceptions; + +/// +/// Indicates that a revision could not be resolved in a repository. +/// +public class CannotResolveRevisionException : ValidationException +{ + public CannotResolveRevisionException(string message) + : base(message) + { + } + + public CannotResolveRevisionException(string message, Exception? cause) + : base(message, cause) + { + } +} diff --git a/src/Copybara.Core/Exceptions/ChangeRejectedException.cs b/src/Copybara.Core/Exceptions/ChangeRejectedException.cs new file mode 100644 index 000000000..a5be3674d --- /dev/null +++ b/src/Copybara.Core/Exceptions/ChangeRejectedException.cs @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Exceptions; + +/// +/// User rejected the change and aborted execution. +/// +public class ChangeRejectedException : ValidationException +{ + public ChangeRejectedException(string message) + : base(message) + { + } +} diff --git a/src/Copybara.Core/Exceptions/CommandLineException.cs b/src/Copybara.Core/Exceptions/CommandLineException.cs new file mode 100644 index 000000000..f5737a390 --- /dev/null +++ b/src/Copybara.Core/Exceptions/CommandLineException.cs @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Exceptions; + +/// +/// An exception due to a command line error. +/// +public class CommandLineException : ValidationException +{ + public CommandLineException(string message) + : base(message) + { + } + + public CommandLineException(string message, Exception? cause) + : base(message, cause) + { + } +} diff --git a/src/Copybara.Core/Exceptions/EmptyChangeException.cs b/src/Copybara.Core/Exceptions/EmptyChangeException.cs new file mode 100644 index 000000000..152d8b9d1 --- /dev/null +++ b/src/Copybara.Core/Exceptions/EmptyChangeException.cs @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Exceptions; + +/// +/// An exception thrown by destinations when they detect that there is no change to submit. Usually +/// this means that the change was already applied. +/// +public class EmptyChangeException : ValidationException +{ + public EmptyChangeException(string message) + : base(message) + { + } + + public EmptyChangeException(Exception? cause, string message) + : base(message, cause) + { + } +} diff --git a/src/Copybara.Core/Exceptions/NonReversibleValidationException.cs b/src/Copybara.Core/Exceptions/NonReversibleValidationException.cs new file mode 100644 index 000000000..86871d7b0 --- /dev/null +++ b/src/Copybara.Core/Exceptions/NonReversibleValidationException.cs @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Starlark.Eval; + +namespace Copybara.Exceptions; + +/// +/// Exception thrown when a Transformation is not reversible but the configuration asked for the +/// reverse. +/// +public class NonReversibleValidationException : EvalException +{ + public NonReversibleValidationException(string message) + : base(message) + { + } + + public NonReversibleValidationException(string message, Exception? cause) + : base(message, cause) + { + } +} diff --git a/src/Copybara.Core/Exceptions/NotADestinationFileException.cs b/src/Copybara.Core/Exceptions/NotADestinationFileException.cs new file mode 100644 index 000000000..9d7469def --- /dev/null +++ b/src/Copybara.Core/Exceptions/NotADestinationFileException.cs @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Exceptions; + +/// +/// An exception that indicates a file that was to be written to a destination is not actually +/// included in destination_files, indicating a bad configuration. +/// +public class NotADestinationFileException : ValidationException +{ + public NotADestinationFileException(string message) + : base(message) + { + } +} diff --git a/src/Copybara.Core/Exceptions/RedundantChangeException.cs b/src/Copybara.Core/Exceptions/RedundantChangeException.cs new file mode 100644 index 000000000..0cf541c2b --- /dev/null +++ b/src/Copybara.Core/Exceptions/RedundantChangeException.cs @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Exceptions; + +/// +/// An exception thrown by destinations when they detect that there is no change to submit, because +/// the same change is already pending in the destination and allow_empty_diff=False was +/// present in the config. +/// +public class RedundantChangeException : EmptyChangeException +{ + public RedundantChangeException(string message, string pendingRevision) + : base(message) + { + PendingRevision = pendingRevision; + } + + public RedundantChangeException(Exception? cause, string message, string pendingRevision) + : base(cause, message) + { + PendingRevision = pendingRevision; + } + + /// + /// A ref (e.g. SHA1) of the pending change that makes the current workflow redundant. + /// + public string PendingRevision { get; } +} diff --git a/src/Copybara.Core/Exceptions/RepoException.cs b/src/Copybara.Core/Exceptions/RepoException.cs new file mode 100644 index 000000000..f0426fb4c --- /dev/null +++ b/src/Copybara.Core/Exceptions/RepoException.cs @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Exceptions; + +/// +/// Exceptions that happen during repository manipulation. +/// +public class RepoException : Exception +{ + public RepoException(string message) + : base(message) + { + } + + public RepoException(string message, Exception? cause) + : base(message, cause) + { + } +} diff --git a/src/Copybara.Core/Exceptions/ValidationException.cs b/src/Copybara.Core/Exceptions/ValidationException.cs new file mode 100644 index 000000000..9b73c5be2 --- /dev/null +++ b/src/Copybara.Core/Exceptions/ValidationException.cs @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text; + +namespace Copybara.Exceptions; + +/// +/// Indicates that the configuration is wrong or some error attributable to the user happened. For +/// example wrong flag usage, errors in fields or errors that we discover during execution. +/// +public class ValidationException : Exception +{ + public ValidationException(string message) + : base(message) + { + } + + public ValidationException(string message, Exception? cause) + : base(message, cause) + { + } + + /// + /// Check a condition and throw if false. + /// + /// if is false + public static void CheckCondition(bool condition, string format, params object?[] args) + { + if (!condition) + { + // Don't try to format if there are no args. This allows strings like '%Fooooo'. + if (args.Length == 0) + { + throw new ValidationException(format); + } + + throw new ValidationException(Format(format, args)); + } + } + + /// + /// Check a condition and throw if false. + /// + /// if is false + public static void CheckCondition(bool condition, string msg) + { + if (!condition) + { + throw new ValidationException(msg); + } + } + + /// + /// Throw a that can be retried. + /// + public static ValidationException RetriableException(string message) + { + return new ValidationException(message); + } + + /// + /// Translates a Java/printf-style format string (using conversions such as %s and + /// %d) into a .NET composite format string and applies . + /// Conversions are consumed sequentially from . A literal percent sign is + /// written using %%. + /// + private static string Format(string fmt, object?[] args) + { + var builder = new StringBuilder(fmt.Length + 16); + var argIndex = 0; + + for (var i = 0; i < fmt.Length; i++) + { + var c = fmt[i]; + if (c == '%' && i + 1 < fmt.Length) + { + var next = fmt[i + 1]; + if (next == '%') + { + builder.Append('%'); + i++; + continue; + } + + // Treat any single-letter conversion (e.g. %s, %d, %b) as a positional argument. + if (char.IsLetter(next)) + { + builder.Append('{').Append(argIndex++).Append('}'); + i++; + continue; + } + } + + // Escape braces so they are treated literally by string.Format. + if (c == '{') + { + builder.Append("{{"); + } + else if (c == '}') + { + builder.Append("}}"); + } + else + { + builder.Append(c); + } + } + + return string.Format(builder.ToString(), args); + } +} diff --git a/src/Copybara.Core/Exceptions/VoidOperationException.cs b/src/Copybara.Core/Exceptions/VoidOperationException.cs new file mode 100644 index 000000000..3ba45dad2 --- /dev/null +++ b/src/Copybara.Core/Exceptions/VoidOperationException.cs @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Exceptions; + +/// +/// An exception that indicates that an operation, as defined, does not do anything to the +/// repository. This usually indicates a problem with the configuration. +/// +public class VoidOperationException : ValidationException +{ + public VoidOperationException(string message) + : base(message) + { + } +} diff --git a/src/Copybara.Core/Feedback/FinishHookContext.cs b/src/Copybara.Core/Feedback/FinishHookContext.cs new file mode 100644 index 000000000..aef21d92d --- /dev/null +++ b/src/Copybara.Core/Feedback/FinishHookContext.cs @@ -0,0 +1,183 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara; +using Copybara.Action; +using Copybara.Common; +using Copybara.Effect; +using Copybara.Exceptions; +using Copybara.Revision; +using Copybara.Transform; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara.Feedback; + +/// Skylark context for 'after migration' hooks. +[StarlarkBuiltin( + "feedback.finish_hook_context", + Doc = + "Gives access to the feedback migration information and utilities. This context is a " + + "concrete implementation for 'after_migration' hooks.")] +public class FinishHookContext : ActionContext +{ + private readonly LazyResourceLoader _origin; + private readonly LazyResourceLoader _destination; + private readonly SkylarkRevision _resolvedRevision; + private readonly ImmutableArray _destinationEffects; + + public FinishHookContext( + IAction action, + LazyResourceLoader origin, + LazyResourceLoader destination, + IReadOnlyList destinationEffects, + IReadOnlyDictionary labels, + IRevision resolvedRevision, + SkylarkConsole console) + : this( + action, + origin, + destination, + destinationEffects, + labels, + console, + Dict.Empty(), + new SkylarkRevision(resolvedRevision)) + { + } + + private FinishHookContext( + IAction currentAction, + LazyResourceLoader origin, + LazyResourceLoader destination, + IReadOnlyList destinationEffects, + IReadOnlyDictionary labels, + SkylarkConsole console, + Dict @params, + SkylarkRevision resolvedRevision) + : base(currentAction, console, labels, @params) + { + _origin = Preconditions.CheckNotNull(origin); + _destination = Preconditions.CheckNotNull(destination); + _destinationEffects = Preconditions.CheckNotNull(destinationEffects).ToImmutableArray(); + _resolvedRevision = resolvedRevision; + } + + [StarlarkMethod( + "origin", + Doc = "An object representing the origin. Can be used to query about the ref or modifying" + + " the origin state", + StructField = true)] + public IEndpoint GetOrigin() + { + try + { + return _origin.Load(Console); + } + catch (Exception e) when (e is RepoException or ValidationException) + { + throw new EvalException(e.Message, e); + } + } + + [StarlarkMethod( + "destination", + Doc = "An object representing the destination. Can be used to query or modify the" + + " destination state", + StructField = true)] + public IEndpoint GetDestination() + { + try + { + return _destination.Load(Console); + } + catch (Exception e) when (e is RepoException or ValidationException) + { + throw new EvalException(e.Message, e); + } + } + + [StarlarkMethod( + "effects", + Doc = "The list of effects that happened in the destination", + StructField = true)] + public StarlarkList GetChanges() => + StarlarkList.ImmutableCopyOf(_destinationEffects.Cast()); + + [StarlarkMethod( + "revision", + Doc = "Get the requested/resolved revision", + StructField = true)] + public SkylarkRevision GetRevision() => _resolvedRevision; + + public override FinishHookContext WithParams(Dict @params) => + new( + Action, _origin, _destination, _destinationEffects, Labels, Console, @params, + _resolvedRevision); + + // Java overrides ActionContext.onFinish to assert the finish hook returned NONE. The ported base + // ActionContext.OnFinish is non-virtual, so this cannot override it and is hidden via `new`. + // TODO(port): make ActionContext.OnFinish virtual so finish-hook dispatch reaches this method + // through ISkylarkContext; until then callers with a static FinishHookContext + // reference get the NONE-result check, and the base still handles effect population. + public new void OnFinish(object? result, object context) + { + ValidationException.CheckCondition( + result == null || Equals(result, StarlarkRt.None), + "Finish hook '{0}' cannot return any result but returned: {1}", + Action.GetName(), + result!); + // Populate effects registered in the action context. This is required because StarlarkAction + // makes a copy of the context to inject the parameters, but that instance is not visible from + // the caller. + if (context is FinishHookContext other) + { + NewDestinationEffects.AddRange(other.GetNewDestinationEffects()); + } + } + + [StarlarkBuiltin( + "feedback.revision_context", + Doc = "Information about the revision request/resolved for the migration")] + public sealed class SkylarkRevision : IStarlarkValue + { + private readonly IRevision _revision; + + internal SkylarkRevision(IRevision revision) + { + _revision = Preconditions.CheckNotNull(revision); + } + + [StarlarkMethod( + "labels", + Doc = "A dictionary with the labels detected for the requested/resolved revision.", + StructField = true)] + public Dict GetLabels() => + Dict.ImmutableCopyOf( + _revision.AssociatedLabels().AsMap().Select( + e => new KeyValuePair( + e.Key, StarlarkList.ImmutableCopyOf(e.Value.Cast())))); + + [StarlarkMethod( + "fill_template", + Doc = "Replaces variables in templates with the values from this revision.")] + public string FillTemplate( + [Param(Name = "template", Doc = "The template to use", Named = true)] string template) => + LabelFinder.MapLabels(label => _revision.AssociatedLabel(label), template); + } +} diff --git a/src/Copybara.Core/FlagAttribute.cs b/src/Copybara.Core/FlagAttribute.cs new file mode 100644 index 000000000..f9833ba42 --- /dev/null +++ b/src/Copybara.Core/FlagAttribute.cs @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara; + +/// +/// Lightweight replacement for JCommander's @Parameter annotation. Marks a property (or +/// field) as a command-line flag, recording the flag names and a description that form the CLI +/// surface. A fully featured parser will consume this metadata later; for now it primarily +/// preserves the flag names/defaults exactly as declared upstream. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, AllowMultiple = false)] +public sealed class FlagAttribute : Attribute +{ + /// The flag names, e.g. --verbose, -v. + public string[] Names { get; } + + /// Human-readable description shown in help output. + public string Description { get; } + + /// Whether the flag is hidden from help output. + public bool Hidden { get; init; } + + /// Number of values the flag consumes (JCommander's arity). Default -1 (unset). + public int Arity { get; init; } = -1; + + public FlagAttribute(string name, string description) + : this(new[] { name }, description) + { + } + + public FlagAttribute(string[] names, string description) + { + Names = names; + Description = description; + } +} diff --git a/src/Copybara.Core/Folder/FolderDestination.cs b/src/Copybara.Core/Folder/FolderDestination.cs new file mode 100644 index 000000000..e2d1819fe --- /dev/null +++ b/src/Copybara.Core/Folder/FolderDestination.cs @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Effect; +using Copybara.Exceptions; +using Copybara.Revision; +using Copybara.Util; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Folder; + +/// +/// Writes the output tree to a local destination. Any file that is not excluded in the configuration +/// gets deleted before writing the new files. +/// +public class FolderDestination : IDestination +{ + private static readonly ILogger Logger = NullLogger.Instance; + + private const string FolderDestinationName = "folder.destination"; + + private static readonly string HistoryNotSupported = + $"History not supported in {FolderDestinationName}. Consider passing a ref as an argument, " + + "or using --last-rev."; + + private readonly GeneralOptions _generalOptions; + private readonly FolderDestinationOptions _folderDestinationOptions; + + internal FolderDestination( + GeneralOptions generalOptions, FolderDestinationOptions folderDestinationOptions) + { + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _folderDestinationOptions = Preconditions.CheckNotNull(folderDestinationOptions); + } + + public IDestination.IWriter NewWriter(WriterContext writerContext) + { + if (writerContext.IsDryRun()) + { + _generalOptions.GetConsole().Warn( + "--dry-run does not have any effect for folder.destination"); + } + return new WriterImpl(this); + } + + private sealed class WriterImpl : IDestination.IWriter + { + private readonly FolderDestination _destination; + + public WriterImpl(FolderDestination destination) => _destination = destination; + + public DestinationStatus? GetDestinationStatus(Glob destinationFiles, string labelName) => + throw new ValidationException(HistoryNotSupported); + + public void VisitChanges(IRevision? start, IChangesVisitor visitor) => + throw new ValidationException(HistoryNotSupported); + + public DestinationReader GetDestinationReader( + Console console, Origin.Baseline? baseline, string workdir) => + GetDestinationReader(console, (string?)"", workdir); + + public DestinationReader GetDestinationReader( + Console console, string? baseline, string workdir) + { + try + { + return new FolderDestinationReader(_destination.GetFolderPath(console), workdir); + } + catch (IOException e) + { + throw new RepoException("Failed to initialize destination reader.", e); + } + } + + public bool SupportsHistory() => false; + + public IReadOnlyList Write( + TransformResult transformResult, Glob destinationFiles, Console console) + { + string localFolder = _destination.GetFolderPath(console); + return WriteToFolder(transformResult, destinationFiles, console, localFolder); + } + } + + public static IReadOnlyList WriteToFolder( + TransformResult transformResult, Glob destinationFiles, Console console, string localFolder) + { + console.Progress("FolderDestination: creating " + localFolder); + bool exists = Directory.Exists(localFolder) || File.Exists(localFolder); + try + { + if (File.Exists(localFolder) && !Directory.Exists(localFolder)) + { + // Mirrors Java Files.createDirectories throwing FileAlreadyExistsException when a + // non-directory already exists at the path. + throw new RepoException( + $"Cannot create '{localFolder}' because '{localFolder}' already exists and is " + + "not a directory"); + } + Directory.CreateDirectory(localFolder); + } + catch (UnauthorizedAccessException e) + { + throw new ValidationException("Path is not accessible: " + localFolder, e); + } + catch (IOException e) + when (e.Message.Contains("Read-only file system") + || e.Message.Contains("Operation not permitted")) + { + throw new ValidationException("Path is not accessible: " + localFolder, e); + } + + console.Progress("FolderDestination: Deleting destination files in " + localFolder); + int numDeletedFiles = FileUtil.DeleteFilesRecursively(localFolder, destinationFiles); + console.Info( + $"FolderDestination: Deleted {numDeletedFiles} existing destination files in {localFolder}"); + + console.Progress("FolderDestination: Copying contents of the workdir to " + localFolder); + FileUtil.CopyFilesRecursively( + transformResult.GetPath(), + localFolder, + FileUtil.CopySymlinkStrategy.FailOutsideSymlinks); + return ImmutableArray.Create( + new DestinationEffect( + exists ? DestinationEffect.EffectType.UPDATED : DestinationEffect.EffectType.CREATED, + $"Folder '{localFolder}' contains the output files of the migration", + transformResult.GetChanges().GetCurrent().Cast().ToList(), + new DestinationEffect.DestinationRef(localFolder, "local_folder", localFolder))); + } + + private string GetFolderPath(Console console) + { + string? localFolderOption = _folderDestinationOptions.LocalFolder; + string localFolder; + if (string.IsNullOrEmpty(localFolderOption)) + { + localFolder = _generalOptions.GetDirFactory().NewTempDir("folder-destination"); + string msg = + "Using folder in default root (--folder-dir to override): " + + Path.GetFullPath(localFolder); + Logger.LogInformation("{Message}", msg); + console.Info(msg); + } + else + { + // Lets assume we are in the same filesystem for now... + localFolder = localFolderOption; + if (!Path.IsPathRooted(localFolder)) + { + localFolder = Path.Combine(_generalOptions.GetCwd(), localFolder); + } + } + + // Normalize for console and other stuff that might require normalized paths + return Path.GetFullPath(localFolder); + } + + public string GetLabelNameWhenOrigin() => + throw new ValidationException(FolderDestinationName + " does not support labels"); + + public string GetTypeName() => "folder.destination"; + + public ImmutableListMultimap Describe(Glob? originFiles) + { + var builder = ImmutableListMultimap.CreateBuilder(); + builder.Put("type", GetTypeName()); + return builder.Build(); + } +} diff --git a/src/Copybara.Core/Folder/FolderDestinationOptions.cs b/src/Copybara.Core/Folder/FolderDestinationOptions.cs new file mode 100644 index 000000000..1114cd406 --- /dev/null +++ b/src/Copybara.Core/Folder/FolderDestinationOptions.cs @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Folder; + +/// Arguments for FolderDestination. +public sealed class FolderDestinationOptions : IOption +{ + [Flag( + "--folder-dir", + "Local directory to write the output of the migration to. If the directory " + + "exists, all files will be deleted. By default Copybara will generate a temporary " + + "directory, so you shouldn't need this.")] + public string? LocalFolder { get; set; } +} diff --git a/src/Copybara.Core/Folder/FolderDestinationReader.cs b/src/Copybara.Core/Folder/FolderDestinationReader.cs new file mode 100644 index 000000000..dac6db02f --- /dev/null +++ b/src/Copybara.Core/Folder/FolderDestinationReader.cs @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Exceptions; +using Copybara.Util; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara.Folder; + +/// A for reading files from a . +public class FolderDestinationReader : DestinationReader +{ + private readonly string _folderPath; + private readonly string _workDir; + + public FolderDestinationReader(string folderPath, string workDir) + { + _folderPath = folderPath; + _workDir = workDir; + } + + public override string ReadFile(string path) + { + try + { + return File.ReadAllText(Path.Combine(_folderPath, path)); + } + catch (IOException e) + { + throw new RepoException($"Unable to read file {path}.", e); + } + } + + public override void CopyDestinationFiles(object glob, object path) + { + CheckoutPath? checkoutPath = + ReferenceEquals(path, StarlarkRt.None) || path is null ? null : (CheckoutPath)path; + Glob resolvedGlob = Glob.WrapGlob(glob, null)!; + if (checkoutPath == null) + { + CopyDestinationFilesToDirectory(resolvedGlob, _workDir); + } + else + { + CopyDestinationFilesToDirectory( + resolvedGlob, + PathOps.Resolve(checkoutPath.GetCheckoutDir(), checkoutPath.GetPath())); + } + } + + public override void CopyDestinationFilesToDirectory(Glob glob, string directory) + { + IPathMatcher pathMatcher = glob.RelativeTo(_folderPath); + + foreach (var root in glob.Roots()) + { + string rootPath = string.IsNullOrEmpty(root) ? _folderPath : Path.Combine(_folderPath, root); + if (!Directory.Exists(rootPath)) + { + continue; + } + try + { + foreach (var sourcePath in Directory.EnumerateFiles( + rootPath, "*", SearchOption.AllDirectories)) + { + string fullSource = Path.GetFullPath(sourcePath); + if (!pathMatcher.Matches(fullSource)) + { + continue; + } + string relative = Path.GetRelativePath(_folderPath, fullSource); + string targetPath = Path.Combine(directory, relative); + string? parent = Path.GetDirectoryName(targetPath); + if (parent != null) + { + Directory.CreateDirectory(parent); + } + File.Copy(fullSource, targetPath); + } + } + catch (IOException e) + { + throw new RepoException($"Failed to copy files from {_folderPath}.", e); + } + } + } + + public override bool Exists(string path) => + File.Exists(Path.Combine(_folderPath, path)) || Directory.Exists(Path.Combine(_folderPath, path)); +} diff --git a/src/Copybara.Core/Folder/FolderModule.cs b/src/Copybara.Core/Folder/FolderModule.cs new file mode 100644 index 000000000..f57e9f75a --- /dev/null +++ b/src/Copybara.Core/Folder/FolderModule.cs @@ -0,0 +1,205 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Authoring; +using Copybara.Common; +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara.Folder; + +/// Main module that groups all the functions related to folders. +[StarlarkBuiltin("folder", Doc = "Module for dealing with local filesystem folders")] +public class FolderModule : IStarlarkValue +{ + private const string DestinationVar = "destination"; + + private readonly FolderOriginOptions _originOptions; + private readonly FolderDestinationOptions _destinationOptions; + private readonly GeneralOptions _generalOptions; + + public FolderModule( + FolderOriginOptions originOptions, + FolderDestinationOptions destinationOptions, + GeneralOptions generalOptions) + { + _originOptions = Preconditions.CheckNotNull(originOptions); + _destinationOptions = Preconditions.CheckNotNull(destinationOptions); + _generalOptions = Preconditions.CheckNotNull(generalOptions); + } + + [StarlarkMethod( + DestinationVar, + Doc = + "A folder destination is a destination that puts the output in a folder. It can be used" + + " both for testing or real production migrations." + + "Given that folder destination does not support a lot of the features of real VCS, " + + "there are some limitations on how to use it:" + + "
    " + + "
  • It requires passing a ref as an argument, as there is no way of calculating " + + "previous migrated changes. Alternatively, --last-rev can be used, which could migrate " + + "N changes." + + "
  • Most likely, the workflow should use 'SQUASH' mode, as history is not supported." + + "
  • If 'ITERATIVE' mode is used, a new temp directory will be created for each change " + + "migrated." + + "
")] + public FolderDestination Destination() => + new(_generalOptions, _destinationOptions); + + [StarlarkMethod( + "origin", + Doc = + "A folder origin is a origin that uses a folder as input. The folder is specified via " + + "the source_ref argument.")] + public FolderOrigin Origin( + [Param( + Name = "materialize_outside_symlinks", + Doc = "DEPRECATED - equivalent to outside_symlinks_mode='MATERIALIZE'", + DefaultValue = "None", + AllowedTypes = new[] { typeof(bool), typeof(NoneType) }, + Named = true)] + object materializeOutsideSymlinks, + [Param( + Name = "inside_symlinks_mode", + Doc = + "How to handle symlinks pointing inside the origin folder. Possible values:" + + " 'COPY_AS_IS' (copy the symlink as-is), 'MATERIALIZE' (copy the content of" + + " the target instead of the symlink), 'IGNORE' (ignore the symlink), 'FAIL'" + + " (fail the operation). Defaults to 'COPY_AS_IS'.", + DefaultValue = "None", + AllowedTypes = new[] { typeof(string), typeof(NoneType) }, + Named = true)] + object insideSymlinksMode, + [Param( + Name = "outside_symlinks_mode", + Doc = + "How to handle symlinks pointing outside the origin folder. See" + + " inside_symlinks_mode for possible values. Defaults to 'FAIL'.", + DefaultValue = "None", + AllowedTypes = new[] { typeof(string), typeof(NoneType) }, + Named = true)] + object outsideSymlinksMode, + [Param( + Name = "broken_symlinks_mode", + Doc = + "How to handle broken symlinks. See inside_symlinks_mode for possible values" + + " (except 'MATERIALIZE', which is invalid for broken symlinks). Defaults to" + + " 'FAIL'.", + DefaultValue = "None", + AllowedTypes = new[] { typeof(string), typeof(NoneType) }, + Named = true)] + object brokenSymlinksMode) + { + bool materializeOutsideSymlinksSet = !ReferenceEquals(materializeOutsideSymlinks, StarlarkRt.None); + bool ignoreInvalidSymlinksSet = _originOptions.IgnoreInvalidSymlinks != null; + bool modernSymlinkOptionsSet = + !ReferenceEquals(insideSymlinksMode, StarlarkRt.None) + || !ReferenceEquals(outsideSymlinksMode, StarlarkRt.None) + || !ReferenceEquals(brokenSymlinksMode, StarlarkRt.None); + + if (materializeOutsideSymlinksSet) + { + _generalOptions + .GetConsole() + .Warn( + "folder.origin(materialize_outside_symlinks = ...) is deprecated. Use" + + " outside_symlinks_mode instead."); + } + if (ignoreInvalidSymlinksSet) + { + _generalOptions + .GetConsole() + .Warn( + "--folder-origin-ignore-invalid-symlinks is deprecated. Use" + + " folder.origin(outside_symlinks_mode = ..., broken_symlinks_mode = ...)" + + " instead."); + } + if (modernSymlinkOptionsSet && (materializeOutsideSymlinksSet || ignoreInvalidSymlinksSet)) + { + throw StarlarkRt.Errorf( + "Cannot mix deprecated symlink configuration ('materialize_outside_symlinks' Starlark" + + " parameter or '--folder-origin-ignore-invalid-symlinks' CLI flag) with new" + + " symlink mode parameters ('inside_symlinks_mode', 'outside_symlinks_mode'," + + " 'broken_symlinks_mode')"); + } + + FileUtil.CopySymlinkStrategy symlinkStrategy; + if (modernSymlinkOptionsSet) + { + try + { + FileUtil.SymlinkMode inside = + ReferenceEquals(insideSymlinksMode, StarlarkRt.None) + ? FileUtil.SymlinkMode.CopyAsIs + : ParseSymlinkMode((string)insideSymlinksMode); + FileUtil.SymlinkMode outside = + ReferenceEquals(outsideSymlinksMode, StarlarkRt.None) + ? FileUtil.SymlinkMode.Fail + : ParseSymlinkMode((string)outsideSymlinksMode); + FileUtil.SymlinkMode broken = + ReferenceEquals(brokenSymlinksMode, StarlarkRt.None) + ? FileUtil.SymlinkMode.Fail + : ParseSymlinkMode((string)brokenSymlinksMode); + symlinkStrategy = new FileUtil.CopySymlinkStrategy(inside, outside, broken); + } + catch (ArgumentException e) + { + throw StarlarkRt.Errorf("Invalid symlink configuration: {0}", e.Message); + } + } + else + { + bool materializeOutside = + materializeOutsideSymlinksSet && (bool)materializeOutsideSymlinks; + bool ignoreInvalid = + ignoreInvalidSymlinksSet && _originOptions.IgnoreInvalidSymlinks!.Value; + symlinkStrategy = + new FileUtil.CopySymlinkStrategy( + inside: FileUtil.SymlinkMode.CopyAsIs, + outside: ignoreInvalid + ? FileUtil.SymlinkMode.Ignore + : (materializeOutside + ? FileUtil.SymlinkMode.Materialize + : FileUtil.SymlinkMode.Fail), + broken: ignoreInvalid ? FileUtil.SymlinkMode.Ignore : FileUtil.SymlinkMode.Fail); + } + + string fs = _generalOptions.GetFileSystem(); + return new FolderOrigin( + fs, + Author.Parse(_originOptions.Author), + _originOptions.Message, + _generalOptions.GetCwd(), + symlinkStrategy, + _originOptions.Version); + } + + /// + /// Parses the Starlark symlink-mode strings ('COPY_AS_IS', 'MATERIALIZE', 'IGNORE', 'FAIL') into + /// the C# enum (whose members are PascalCase). + /// + private static FileUtil.SymlinkMode ParseSymlinkMode(string value) => + value switch + { + "COPY_AS_IS" => FileUtil.SymlinkMode.CopyAsIs, + "MATERIALIZE" => FileUtil.SymlinkMode.Materialize, + "IGNORE" => FileUtil.SymlinkMode.Ignore, + "FAIL" => FileUtil.SymlinkMode.Fail, + _ => throw new ArgumentException($"No enum constant for symlink mode: {value}"), + }; +} diff --git a/src/Copybara.Core/Folder/FolderOrigin.cs b/src/Copybara.Core/Folder/FolderOrigin.cs new file mode 100644 index 000000000..92d75d1bd --- /dev/null +++ b/src/Copybara.Core/Folder/FolderOrigin.cs @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Authoring; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Revision; +using Copybara.Util; + +namespace Copybara.Folder; + +/// Use a folder as the input for the migration. +public class FolderOrigin : IOrigin +{ + private const string LabelName = "FolderOrigin-RevId"; + + private const UnixFileMode FilePermissions = UnixFileMode.UserRead | UnixFileMode.UserWrite; + + private readonly string _fileSystemRoot; + private readonly Author _author; + private readonly string _message; + private readonly string _cwd; + private readonly FileUtil.CopySymlinkStrategy _copySymlinkStrategy; + private readonly string? _version; + + internal FolderOrigin( + string fileSystemRoot, + Author author, + string message, + string cwd, + FileUtil.CopySymlinkStrategy copySymlinkStrategy, + string? version) + { + _fileSystemRoot = Preconditions.CheckNotNull(fileSystemRoot); + _author = author; + _message = message; + _cwd = Preconditions.CheckNotNull(cwd); + _copySymlinkStrategy = Preconditions.CheckNotNull(copySymlinkStrategy); + _version = version; + } + + public FolderRevision Resolve(string? reference) + { + ValidationException.CheckCondition( + reference != null, + "A path is expected as reference in the command line. Invoke copybara as:\n" + + " copybara copy.bara.sky workflow_name ORIGIN_FOLDER"); + string path = reference!; + if (!Path.IsPathRooted(path)) + { + path = Path.GetFullPath(Path.Combine(_cwd, path)); + } + ValidationException.CheckCondition( + Directory.Exists(path) || File.Exists(path), "%s folder doesn't exist", path); + ValidationException.CheckCondition(Directory.Exists(path), "%s is not a folder", path); + + return new FolderRevision(path, DateTimeOffset.Now, _version); + } + + public IOrigin.IReader NewReader( + Glob originFiles, Authoring.Authoring authoring) => + new ReaderImpl(this, originFiles); + + public string GetLabelName() => LabelName; + + public string GetTypeName() => "folder.origin"; + + public ImmutableListMultimap Describe(Glob? originFiles) + { + var builder = ImmutableListMultimap.CreateBuilder(); + builder.Put("type", GetTypeName()); + return builder.Build(); + } + + private sealed class ReaderImpl : IOrigin.IReader + { + private readonly FolderOrigin _origin; + private readonly Glob _originFiles; + + public ReaderImpl(FolderOrigin origin, Glob originFiles) + { + _origin = origin; + _originFiles = originFiles; + } + + public void Checkout(FolderRevision reference, string checkoutDir) + { + try + { + FileUtil.CopyFilesRecursively( + reference.Path, checkoutDir, _origin._copySymlinkStrategy, _originFiles); + FileUtil.AddPermissionsAllRecursively(checkoutDir, FilePermissions); + } + catch (SymlinkException e) + { + throw new ValidationException( + "Cannot copy files into the workdir: " + e.Message, e); + } + catch (IOException e) + { + throw new RepoException( + "Cannot copy files into the workdir:\n" + + $" origin folder: {reference.Path}\n" + + $" workdir: {checkoutDir}", + e); + } + } + + public Origin.ChangesResponse Changes( + FolderRevision? fromRef, FolderRevision toRef) + { + // Ignore fromRef since a folder doesn't have history of changes + return Origin.ChangesResponse.ForChanges( + ImmutableArray.Create(Change(toRef))); + } + + public bool SupportsHistory() => false; + + public Change Change(FolderRevision reference) => + new( + reference, + _origin._author, + _origin._message, + reference.ReadTimestamp() ?? DateTimeOffset.Now, + ImmutableListMultimap.Empty); + + public void VisitChanges(FolderRevision? start, IChangesVisitor visitor) + { + FolderRevision reference = start!; + var change = new Change( + reference, + _origin._author, + _origin._message, + reference.ReadTimestamp() ?? DateTimeOffset.Now, + ImmutableListMultimap.Empty); + visitor.Visit(change); + } + } +} diff --git a/src/Copybara.Core/Folder/FolderOriginOptions.cs b/src/Copybara.Core/Folder/FolderOriginOptions.cs new file mode 100644 index 000000000..9a30ac44e --- /dev/null +++ b/src/Copybara.Core/Folder/FolderOriginOptions.cs @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Folder; + +/// Arguments for FolderOrigin. +public sealed class FolderOriginOptions : IOption +{ + [Flag( + "--folder-origin-author", + "Deprecated. Please use '--force-author'." + + " Author of the change being migrated from folder.origin()")] + public string Author { get; set; } = "Copybara "; + + [Flag( + "--folder-origin-message", + "Deprecated. Please use '--force-message'. Message of the change being migrated" + + " from folder.origin()")] + public string Message { get; set; } = "Copybara code migration"; + + [Flag( + "--folder-origin-version", + "The version string associated with the change migrated from folder.origin(). If not" + + " specified, the default will be the folder path.")] + public string? Version { get; set; } + + [Flag( + "--folder-origin-ignore-invalid-symlinks", + "DEPRECATED - equivalent to folder.origin(outside_symlinks_mode='IGNORE'," + + " broken_symlinks_mode='IGNORE')", + Arity = 1)] + public bool? IgnoreInvalidSymlinks { get; set; } +} diff --git a/src/Copybara.Core/Folder/FolderRevision.cs b/src/Copybara.Core/Folder/FolderRevision.cs new file mode 100644 index 000000000..59b5e8efb --- /dev/null +++ b/src/Copybara.Core/Folder/FolderRevision.cs @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Copybara.Revision; + +namespace Copybara.Folder; + +/// A reference for folder origins. +public class FolderRevision : IRevision +{ + internal readonly string Path; + private readonly DateTimeOffset _timestamp; + internal readonly string? Version; + + public FolderRevision(string path, DateTimeOffset timestamp) + { + Preconditions.CheckState(System.IO.Path.IsPathRooted(path), "Path must be absolute"); + Path = path; + _timestamp = timestamp; + Version = null; + } + + internal FolderRevision(string path, DateTimeOffset timestamp, string? version) + { + Preconditions.CheckState(System.IO.Path.IsPathRooted(path), "Path must be absolute"); + Path = path; + _timestamp = timestamp; + Version = version; + } + + public string AsString() => Version ?? Path; + + public DateTimeOffset? ReadTimestamp() => _timestamp; +} diff --git a/src/Copybara.Core/Format/BuildifierFormat.cs b/src/Copybara.Core/Format/BuildifierFormat.cs new file mode 100644 index 000000000..24c0627b0 --- /dev/null +++ b/src/Copybara.Core/Format/BuildifierFormat.cs @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Util; +using Copybara.Util.Console; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Command = Copybara.Util.Command; +using CommandException = Copybara.Util.CommandException; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Format; + +/// Format using buildifier. +public class BuildifierFormat : ITransformation +{ + private static readonly ILogger Logger = NullLogger.Instance; + + private readonly BuildifierOptions _buildifierOptions; + private readonly GeneralOptions _generalOptions; + private readonly Glob _glob; + private readonly LintMode _lintMode; + private readonly ImmutableArray _warnings; + private readonly string? _type; + + internal BuildifierFormat( + BuildifierOptions buildifierOptions, + GeneralOptions generalOptions, + Glob glob, + LintMode lintMode, + ImmutableArray warnings, + string? type) + { + _buildifierOptions = Preconditions.CheckNotNull(buildifierOptions); + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _glob = Preconditions.CheckNotNull(glob); + _lintMode = lintMode; + _warnings = warnings; + _type = type; + } + + public TransformationStatus Transform(TransformWork work) + { + string checkoutDir = work.GetCheckoutDir(); + IPathMatcher pathMatcher = _glob.RelativeTo(checkoutDir); + var paths = ImmutableArray.CreateBuilder(); + foreach (string file in Directory.EnumerateFiles( + checkoutDir, "*", SearchOption.AllDirectories)) + { + if (pathMatcher.Matches(file)) + { + paths.Add(Path.GetFullPath(file)); + } + } + + ImmutableArray builtPaths = paths.ToImmutable(); + if (builtPaths.Length == 0) + { + return TransformationStatus.Noop(_glob + " didn't match any build file to format"); + } + + for (int i = 0; i < builtPaths.Length; i += _buildifierOptions.BatchSize) + { + var sublist = builtPaths.Skip(i).Take(_buildifierOptions.BatchSize).ToList(); + Run(work.GetConsole(), checkoutDir, sublist); + } + + return TransformationStatus.Success(); + } + + /// Runs buildifier with the given arguments. + private void Run(Console console, string checkoutDir, IReadOnlyList args) + { + var argBuilder = new List { _buildifierOptions.BuildifierBin }; + if (_type != null) + { + argBuilder.Add("-type=" + _type); + } + if (_lintMode != LintMode.Off) + { + argBuilder.Add("-lint=" + _lintMode.ToString().ToLowerInvariant()); + if (!_warnings.IsDefaultOrEmpty) + { + argBuilder.Add("-warnings=" + string.Join(",", _warnings)); + } + } + argBuilder.AddRange(args); + + try + { + var cmd = new Command(argBuilder.ToArray(), null, checkoutDir); + CommandOutputWithStatus output = _generalOptions.NewCommandRunner(cmd) + .WithVerbose(_generalOptions.IsVerbose()) + .Execute(); + if (output.GetStdout().Length != 0) + { + Logger.LogInformation("buildifier stdout: {Stdout}", output.GetStdout()); + } + if (output.GetStderr().Length != 0) + { + Logger.LogInformation("buildifier stderr: {Stderr}", output.GetStderr()); + } + } + catch (BadExitStatusWithOutputException e) + { + Log(console, e.GetOutput()); + ValidationException.CheckCondition( + e.GetResult().TerminationStatus.GetExitCode() != 1, + "Build file(s) couldn't be formatted because there was a syntax error"); + throw new IOException( + "Failed to execute buildifier with args: " + string.Join(" ", args), e); + } + catch (CommandException e) + { + throw new IOException( + "Failed to execute buildifier with args: " + string.Join(" ", args), e); + } + } + + private static void Log(Console console, CommandOutput output) + { + Consoles.LogLines(console, "buildifier stdout: ", output.GetStdout()); + Consoles.LogLines(console, "buildifier stderr: ", output.GetStderr()); + } + + public ITransformation Reverse() => this; + + public string Describe() => "Buildifier"; + + /// Valid modes that we support for buildifier -lint flag. + public enum LintMode + { + Off, + + // Warn, // Not exposed for now since we don't show the stderr/out warnings in the console + Fix, + } +} diff --git a/src/Copybara.Core/Format/BuildifierOptions.cs b/src/Copybara.Core/Format/BuildifierOptions.cs new file mode 100644 index 000000000..5b1800e15 --- /dev/null +++ b/src/Copybara.Core/Format/BuildifierOptions.cs @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Format; + +/// Specifies how Buildifier is executed. +public class BuildifierOptions : IOption +{ + [Flag( + "--buildifier-bin", + "Binary to use for buildifier (Default is /usr/bin/buildifier)", + Hidden = true)] + public string BuildifierBin { get; set; } = "/usr/bin/buildifier"; + + [Flag("--buildifier-batch-size", "Process files in batches this size")] + public int BatchSize { get; set; } = 200; +} diff --git a/src/Copybara.Core/Format/FormatModule.cs b/src/Copybara.Core/Format/FormatModule.cs new file mode 100644 index 000000000..94dcfe08d --- /dev/null +++ b/src/Copybara.Core/Format/FormatModule.cs @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Config; +using Copybara.Format; +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; +using StarlarkRt = Starlark.Eval.Starlark; + +namespace Copybara.Format; + +/// Skylark module for transforming the code to Google's style/guidelines. +[StarlarkBuiltin( + "format", + Doc = "Module for formatting the code to Google's style/guidelines")] +public class FormatModule : IStarlarkValue +{ + private static readonly ImmutableHashSet BuildifierTypeValues = + ImmutableHashSet.Create("auto", "bzl", "build", "workspace"); + + private static readonly Glob DefaultBuildifierPaths = + Glob.CreateGlob(ImmutableArray.Create("**.bzl", "**/BUILD", "BUILD")); + + protected readonly WorkflowOptions WorkflowOptions; + protected readonly BuildifierOptions BuildifierOptions; + protected readonly GeneralOptions GeneralOptions; + + public FormatModule( + WorkflowOptions workflowOptions, + BuildifierOptions buildifierOptions, + GeneralOptions generalOptions) + { + WorkflowOptions = Preconditions.CheckNotNull(workflowOptions); + BuildifierOptions = Preconditions.CheckNotNull(buildifierOptions); + GeneralOptions = Preconditions.CheckNotNull(generalOptions); + } + + [StarlarkMethod( + "buildifier", + Doc = "Formats the BUILD files using buildifier.")] + public ITransformation Buildifier( + [Param( + Name = "paths", + AllowedTypes = new[] { typeof(Glob), typeof(StarlarkList), typeof(NoneType) }, + Doc = "Paths of the files to format relative to the workdir.", + DefaultValue = "None", + Named = true)] + object? paths, + [Param( + Name = "type", + AllowedTypes = new[] { typeof(string), typeof(NoneType) }, + Doc = + "The type of the files. Can be 'auto', 'bzl', 'build' or 'workspace'. Note that" + + " this is not recommended to be set and might break in the future. The" + + " default is 'auto'. This mode formats as BUILD files \"BUILD\"," + + " \"BUILD.bazel\", \"WORKSPACE\" and \"WORKSPACE.bazel\" files. The rest as" + + " bzl files. Prefer to use those names for BUILD files instead of setting" + + " this flag.", + DefaultValue = "'auto'", + Named = true)] + object? type, + [Param( + Name = "lint", + AllowedTypes = new[] { typeof(string), typeof(NoneType) }, + Doc = + "If buildifier --lint should be used. This fixes several common issues. Note that" + + " this transformation is difficult to revert. For example if it removes a" + + " load statement because is not used after removing a rule, then the reverse" + + " workflow needs to add back the load statement (core.replace or similar). " + + " Possible values: `OFF`, `FIX`. Default is `OFF`", + DefaultValue = "None", + Named = true)] + object? lint, + [Param( + Name = "lint_warnings", + AllowedTypes = new[] { typeof(Sequence) }, + DefaultValue = "[]", + Doc = "Warnings used in the lint mode. Default is buildifier default", + Named = true)] + object warnings) + { + string? typeStr = SkylarkUtil.ConvertFromNoneable(type, null); + if (typeStr != null) + { + SkylarkUtil.Check( + BuildifierTypeValues.Contains(typeStr), + "Non-valid type: {0}. Valid types: {1}", + typeStr, + string.Join(", ", BuildifierTypeValues)); + } + + string lintStr = SkylarkUtil.ConvertFromNoneable(lint, "OFF")!; + BuildifierFormat.LintMode lintMode = ParseLintMode(lintStr); + + var warningsList = SkylarkUtil.ConvertStringList(warnings, "lint_warnings"); + SkylarkUtil.Check( + lintMode != BuildifierFormat.LintMode.Off || warningsList.Count == 0, + "Warnings can only be used when lint is set to FIX"); + + return new BuildifierFormat( + BuildifierOptions, + GeneralOptions, + Glob.WrapGlob(paths, DefaultBuildifierPaths)!, + lintMode, + warningsList.ToImmutableArray(), + typeStr); + } + + private static BuildifierFormat.LintMode ParseLintMode(string value) + { + if (Enum.TryParse(value, ignoreCase: true, out var result) + && Enum.IsDefined(result)) + { + return result; + } + + throw StarlarkRt.Errorf( + "Invalid value '{0}' for field 'lint'. Valid values are: {1}", + value, + string.Join(", ", Enum.GetNames())); + } +} diff --git a/src/Copybara.Core/GeneralOptions.cs b/src/Copybara.Core/GeneralOptions.cs new file mode 100644 index 000000000..56babf509 --- /dev/null +++ b/src/Copybara.Core/GeneralOptions.cs @@ -0,0 +1,508 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Util; +using Copybara.Util.Console; + +// Domain 'Console' collides with System.Console; qualify both. +using Console = Copybara.Util.Console.Console; + +namespace Copybara; + +/// General options available for all the program classes. +public sealed class GeneralOptions : IOption +{ + public const string CliFlagPrefix = "FLAG_"; + + public const string Noansi = "--noansi"; + public const string Noprompt = "--noprompt"; + public const string Force = "--force"; + public const string ConfigRootFlag = "--config-root"; + public const string OutputRootFlag = "--output-root"; + public const string OutputLimitFlag = "--output-limit"; + public const string DryRunFlag = "--dry-run"; + public const string SquashFlag = "--squash"; + public const string PatchBinFlag = "--patch-bin"; + internal const string ConsoleFilePath = "--console-file-path"; + internal const string ConsoleFileFlushInterval = "--console-file-flush-interval"; + + internal static readonly TimeSpan DefaultConsoleFileFlushInterval = TimeSpan.FromSeconds(30); + private const string DefaultMonitor = "default"; + + private IReadOnlyDictionary _environment; + private string _fileSystemRoot; + private Console _console; + + // Registry of available event monitors, keyed by name. Populated lazily on first access because + // the default monitor needs the console. Mirrors Java's HashMap. + private Dictionary? _eventMonitors; + + private string? _configRootPath; + private string? _outputRootPath; + + private Copybara.Profiler.Profiler _profiler = new(Copybara.Profiler.Ticker.SystemTicker); + + public GeneralOptions( + IReadOnlyDictionary environment, + string fileSystemRoot, + Console console) + { + _environment = environment; + _fileSystemRoot = Preconditions.CheckNotNull(fileSystemRoot); + _console = Preconditions.CheckNotNull(console); + } + + public GeneralOptions( + IReadOnlyDictionary environment, + string fileSystemRoot, + bool verbose, + Console console, + string? configRoot, + string? outputRoot, + bool noCleanup, + bool disableReversibleCheck, + bool force, + int outputLimit) + : this(environment, fileSystemRoot, console) + { + Verbose = verbose; + _configRootPath = configRoot; + _outputRootPath = outputRoot; + NoCleanup = noCleanup; + DisableReversibleCheck = disableReversibleCheck; + ForceFlag = force; + OutputLimitValue = outputLimit; + } + + public GeneralOptions WithForce(bool force) + { + return new GeneralOptions( + _environment, _fileSystemRoot, Verbose, _console, GetConfigRoot(), + GetOutputRoot(), NoCleanup, DisableReversibleCheck, force, OutputLimitValue); + } + + public GeneralOptions WithConsole(Console console) + { + return new GeneralOptions( + _environment, _fileSystemRoot, Verbose, console, GetConfigRoot(), + GetOutputRoot(), NoCleanup, DisableReversibleCheck, ForceFlag, OutputLimitValue); + } + + public IReadOnlyDictionary GetEnvironment() => _environment; + + public bool IsVerbose() => Verbose; + + public Console GetConsole() => _console; + + /// The root of the filesystem used to resolve paths (analog of Java's FileSystem). + public string GetFileSystem() => _fileSystemRoot; + + public bool IsNoCleanup() => NoCleanup; + + public bool IsDisableReversibleCheck() => DisableReversibleCheck; + + public bool IsForced() => ForceFlag; + + public bool IsVersionSelectorUseCliRef() => VersionSelectorUseCliRef; + + /// Returns current working directory. + public string GetCwd() + { + return _environment.TryGetValue("PWD", out var pwd) + ? pwd + : Directory.GetCurrentDirectory(); + } + + /// Returns the root absolute path to use for config, or null if not set. + public string? GetConfigRoot() + { + if (_configRootPath == null && ConfigRoot != null) + { + _configRootPath = Path.GetFullPath(ConfigRoot); + ValidationException.CheckCondition( + Directory.Exists(_configRootPath) || File.Exists(_configRootPath), + "%s doesn't exist", ConfigRoot); + ValidationException.CheckCondition( + Directory.Exists(_configRootPath), "%s isn't a directory", ConfigRoot); + } + + return _configRootPath; + } + + /// + /// Returns the output root directory, or null if not set. + /// This method is exposed mainly for tests; prefer . + /// + public string? GetOutputRoot() + { + if (_outputRootPath == null && OutputRoot != null) + { + _outputRootPath = OutputRoot; + } + + return _outputRootPath; + } + + /// + /// Returns the output limit. + /// Each subcommand can use this value differently. + /// + public int GetOutputLimit() => OutputLimitValue > 0 ? OutputLimitValue : int.MaxValue; + + public Copybara.Profiler.Profiler Profiler() => _profiler; + + private Dictionary EventMonitorRegistry() + { + // Default registry contains the "default" monitor: a ConsoleEventMonitor wrapping the empty + // monitor. Mirrors GeneralOptions' constructor in Java. + return _eventMonitors ??= new Dictionary + { + [DefaultMonitor] = + new Copybara.Monitor.ConsoleEventMonitor( + _console, Copybara.Monitor.IEventMonitor.EmptyMonitor), + }; + } + + /// + /// Returns the configured (enabled) event monitors. Mirrors Java's eventMonitors(): it + /// filters the available-monitor registry down to those named in . + /// + public Copybara.Monitor.IEventMonitor.EventMonitors EventMonitors() + { + var registry = EventMonitorRegistry(); + var result = new List(); + foreach (var name in EnabledEventMonitors) + { + if (registry.TryGetValue(name, out var monitor)) + { + result.Add(monitor); + } + } + + return new Copybara.Monitor.IEventMonitor.EventMonitors(result); + } + + /// + /// Adds an EventMonitor to the list of available monitors without activating it. Use this to make + /// a monitor available for later enabling. + /// + public GeneralOptions AddEventMonitor(string name, Copybara.Monitor.IEventMonitor eventMonitor) + { + EventMonitorRegistry()[name] = eventMonitor; + return this; + } + + /// Enables an already-registered EventMonitor by name. + public GeneralOptions EnableEventMonitor(string name) + { + ValidationException.CheckCondition( + EventMonitorRegistry().ContainsKey(name), "%s is not a known EventMonitor.", name); + EnabledEventMonitors.Add(name); + return this; + } + + /// Adds an EventMonitor to the list of available monitors and enables it. + public GeneralOptions EnableEventMonitor(string name, Copybara.Monitor.IEventMonitor eventMonitor) + { + AddEventMonitor(name, eventMonitor); + EnableEventMonitor(name); + return this; + } + + /// Clears the enabled event monitors. + public GeneralOptions ClearEventMonitor() + { + EnabledEventMonitors.Clear(); + return this; + } + + public IReadOnlyDictionary CliLabels() => Labels; + + /// Run a repository task with profiling. + public T RepoTask(string description, Func callable) + { + using (_profiler.Start(description)) + { + return callable(); + } + } + + /// Run a repository task that can throw IOException with profiling. + public T IoRepoTask(string description, Func callable) + { + using (_profiler.Start(description)) + { + return callable(); + } + } + + /// + /// Returns a capable of creating directories in a self contained + /// location in the filesystem. + /// By default, the directories are created under $HOME/copybara, but it can be + /// overridden with the flag --output-root. + /// + public DirFactory GetDirFactory() + { + var outputRoot = GetOutputRoot(); + if (outputRoot != null) + { + return new DirFactory(outputRoot); + } + + var home = _environment.TryGetValue("HOME", out var h) ? h : null; + Preconditions.CheckNotNull(home, "$HOME environment var is not set"); + return new DirFactory(Path.Combine(home!, "copybara")); + } + + public void SetEnvironmentForTest(IReadOnlyDictionary environment) + => _environment = environment; + + public void SetTemporaryFeaturesForTest(IReadOnlyDictionary temporaryFeatures) + => TemporaryFeatures = temporaryFeatures.ToImmutableDictionary(); + + public void SetOutputRootPathForTest(string outputRootPath) => _outputRootPath = outputRootPath; + + public void SetConsoleForTest(Console console) => _console = console; + + public void SetForceForTest(bool force) => ForceFlag = force; + + public void SetVersionSelectorUseCliRefForTest(bool versionSelectorUseCliRef) + => VersionSelectorUseCliRef = versionSelectorUseCliRef; + + public void SetCliLabelsForTest(ImmutableDictionary labels) => Labels = labels; + + public void SetFileSystemForTest(string fileSystemRoot) => _fileSystemRoot = fileSystemRoot; + + public void SetSquashForTest(bool squash) => Squash = squash; + + public GeneralOptions WithProfiler(Copybara.Profiler.Profiler profiler) + { + _profiler = Preconditions.CheckNotNull(profiler); + return this; + } + + // ---- Flags ---- + + [Flag(new[] { "-v", "--verbose" }, "Verbose output.")] + internal bool Verbose { get; set; } + + [Flag("--repo-timeout", "Repository operation timeout duration.")] + public TimeSpan RepoTimeout { get; set; } = CommandRunner.DefaultTimeout; + + [Flag("--commands-timeout", "Commands timeout")] + public TimeSpan CommandsTimeout { get; set; } = CommandRunner.DefaultTimeout; + + public CommandRunner NewCommandRunner(Command cmd) => new(cmd, CommandsTimeout); + + // We don't use JCommander for parsing these flags but we do it manually since + // the parsing could fail and we need to report errors using one console. + [Flag(Noansi, "Don't use ANSI output for messages")] + internal bool Noansi_ { get; set; } + + [Flag(Noprompt, "Don't prompt, this will answer all prompts with 'yes'", Arity = 1)] + internal bool NoPrompt { get; set; } + + [Flag( + new[] { Force, "--force-update" }, + "Force the migration even if Copybara cannot find in the destination a change that is an" + + " ancestor of the one(s) being migrated. This should be used with care, as it" + + " could lose changes when migrating a previous/conflicting change.")] + internal bool ForceFlag { get; set; } + + [Flag( + "--version-selector-use-cli-ref", + "If command line ref is to used with a version selector, pass this flag to tell copybara" + + " to use it.", + Arity = 1)] + internal bool VersionSelectorUseCliRef { get; set; } = true; + + [Flag( + ConfigRootFlag, + "Configuration root path to be used for resolving absolute config labels" + + " like '//foo/bar'")] + internal string? ConfigRoot { get; set; } + + [Flag( + "--disable-reversible-check", + "If set, all workflows will be executed without reversible_check, overriding" + + " the workflow config and the normal behavior for CHANGE_REQUEST mode.")] + internal bool DisableReversibleCheck { get; set; } + + [Flag( + OutputRootFlag, + "The root directory where to generate output files. If not set, ~/copybara/out is used " + + "by default. Use with care, Copybara might remove files inside this root if " + + "necessary.")] + internal string? OutputRoot { get; set; } + + [Flag( + OutputLimitFlag, + "Limit the output in the console to a number of records. Each subcommand might use this " + + "flag differently. Defaults to 0, which shows all the output.")] + internal int OutputLimitValue { get; set; } + + [Flag( + "--nocleanup", + "Cleanup the output directories. This includes the workdir, scratch clones of Git" + + " repos, etc. By default is set to false and directories will be cleaned prior to" + + " the execution. If set to true, the previous run output will not be cleaned up." + + " Keep in mind that running in this mode will lead to an ever increasing disk" + + " usage.")] + internal bool NoCleanup { get; set; } + + [Flag( + "--nologging", + "Disable logging of this binary. Note that commands executed by Copybara " + + "might still log to their own file.", + Hidden = true)] + internal bool NoLogging { get; set; } + + [Flag( + "--labels", + "Additional flags. Can be accessed in feedback and mirror context objects via the" + + " `cli_labels` field. In `core.workflow`, they are accessible as labels, but with" + + " names uppercased and prefixed with " + + CliFlagPrefix + + " to avoid name clashes with existing labels. I.e. `--labels=label1:value1` will" + + " define a label FLAG_LABEL1Format: --labels=flag1:value1,flag2:value2 Or: --labels" + + " flag1:value1,flag2:value2 ")] + internal ImmutableDictionary Labels { get; set; } = + ImmutableDictionary.Empty; + + [Flag( + "--temporary-features", + "Change guarded features. If set it means that it will return true.", + Hidden = true)] + private ImmutableDictionary TemporaryFeatures { get; set; } = + ImmutableDictionary.Empty; + + [Flag( + "--diff-bin", + "Command line diff tool bin used in merge import. Defaults to diff3, but users can pass" + + " in their own diffing tools (along with requisite arg reordering)")] + private string DiffBin { get; set; } = "diff3"; + + public string GetDiffBin() => DiffBin; + + [Flag(PatchBinFlag, "Path for GNU Patch command")] + public string PatchBin { get; set; } = "patch"; + + /// + /// Temporary features is meant to be used by Copybara team for guarding new codepaths. Should + /// never be used for user facing flags or longer term experiments. Any caller of this function + /// should have a todo saying when to remove the call. + /// If the flag doesn't have a value it will use defaultVal. If the flag is incorrect + /// (different from true/false) it will use defaultVal (and log at severe). + /// + public bool IsTemporaryFeature(string name, bool defaultVal) + { + Preconditions.CheckNotNull(name); + if (!TemporaryFeatures.TryGetValue(name, out var v)) + { + return defaultVal; + } + + if (string.Equals(v, "true", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (string.Equals(v, "false", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + // logger.atSevere(): Invalid boolean value. Using default. + return defaultVal; + } + + // This flag is read before we parse the arguments, because of the console lifecycle. + [Flag( + ConsoleFilePath, + "If set, write the console output also to the given file path.")] + internal string? ConsoleFilePathValue { get; set; } + + // This flag is read before we parse the arguments, because of the console lifecycle. + [Obsolete] + [Flag( + "--console-file-flush-rate", + "How often in number of lines to flush the console to the output file. " + + "If set to 0, console will be flushed only at the end.", + Hidden = true)] + internal int ConsoleFileFlushRateDeprecatedDontUse { get; set; } = -1; + + // This flag is read before we parse the arguments, because of the console lifecycle. + [Flag( + ConsoleFileFlushInterval, + "How often Copybara should flush the console to the output file. (10s, 1m, etc.)" + + "If set to 0s, console will be flushed only at the end.")] + internal TimeSpan ConsoleFileFlushIntervalValue { get; set; } = DefaultConsoleFileFlushInterval; + + [Flag( + DryRunFlag, + "Run the migration in dry-run mode. Some destination implementations might" + + " have some side effects (like creating a code review), but never submit to a main" + + " branch.")] + public bool DryRunMode { get; set; } + + [Flag( + SquashFlag, + "Override workflow's mode with 'SQUASH'. This is " + + "useful mainly for workflows that use 'ITERATIVE' mode, when we want to run a single " + + "export with 'SQUASH', maybe to fix an issue. Always use " + DryRunFlag + " before, to " + + "test your changes locally.")] + public bool Squash { get; set; } + + [Flag( + "--validate-starlark", + "Starlark should be validated prior to execution, but this might break legacy configs." + + " Options are LOOSE, STRICT")] + public string StarlarkModeFlag { get; set; } = StarlarkMode.Loose.ToString().ToUpperInvariant(); + + public StarlarkMode GetStarlarkMode() => + Enum.Parse(StarlarkModeFlag, ignoreCase: true); + + [Flag( + "--info-list-only", + "When set, the INFO command will print a list of workflows defined in the file.")] + public bool InfoListOnly { get; set; } + + [Flag( + "--info-include-definition", + "When set, the INFO command will include the migrations' definition stack info in the" + + " table or list output. In table, leaves out origin, destination and mode.")] + public bool InfoIncludeDefinition { get; set; } + + [Flag( + "--event-monitor", + "Eventmonitors to enable. These must be in the list of available monitors.")] + public List EnabledEventMonitors { get; set; } = new() { DefaultMonitor }; + + [Flag( + "--allow-empty-diff", + "If set to false, Copybara will not write to the destination if the exact same change is" + + " already pending in the destination. Currently only supported for" + + " `git.github_pr_destination` and `git.gerrit_destination`.", + Arity = 1)] + public bool? AllowEmptyDiff { get; set; } + + public bool AllowEmptyDiffValue(bool configAllowEmptyDiff) + => AllowEmptyDiff ?? configAllowEmptyDiff; +} diff --git a/src/Copybara.Core/Git/AddExcludedFilesToIndex.cs b/src/Copybara.Core/Git/AddExcludedFilesToIndex.cs new file mode 100644 index 000000000..5db263cc3 --- /dev/null +++ b/src/Copybara.Core/Git/AddExcludedFilesToIndex.cs @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.RegularExpressions; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Util; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Git; + +/// +/// A walker which adds all files not matching a glob to the index of a Git repo using +/// git add. Port of com.google.copybara.git.AddExcludedFilesToIndex. +/// +internal sealed class AddExcludedFilesToIndex +{ + private static readonly Regex SubmoduleStatusPrefix = new("^-[0-9a-f]{40,64} ", RegexOptions.Compiled); + + private readonly GitRepository _repo; + private readonly IPathMatcher _pathMatcher; + private readonly string _workTree; + private List? _addBackSubmodules; + + // Relative paths (using '/' separators) to add back to the index. Sorted for stable batching. + private readonly SortedSet _toExclude = new(StringComparer.Ordinal); + + internal AddExcludedFilesToIndex(GitRepository repo, IPathMatcher pathMatcher) + { + _repo = Preconditions.CheckNotNull(repo); + _workTree = Preconditions.CheckNotNull(repo.GetWorkTree()); + _pathMatcher = Preconditions.CheckNotNull(pathMatcher); + } + + internal void Prepare(string workdir) + { + var included = new HashSet(StringComparer.Ordinal); + var prevExcluded = new List(); + IReadOnlyList head; + try + { + head = _repo.LsTree(_repo.ResolveReference("HEAD"), null, true, true); + } + catch (CannotResolveRevisionException) + { + // Destination is empty. Nothing to revert. + return; + } + foreach (var treeElement in head) + { + string relative = Normalize(treeElement.Path); + if (_pathMatcher.Matches(Path.Combine(_workTree, treeElement.Path))) + { + AddPathAndParents(included, relative); + } + else + { + prevExcluded.Add(relative); + if (IsHidden(relative)) + { + // File is not included but 'git add dir' doesn't work for 'dir/.file'. + AddPathAndParents(included, Parent(relative)); + } + } + } + + foreach (var file in EnumerateRelativeFiles(workdir)) + { + AddPathAndParents(included, file); + } + + foreach (var path in prevExcluded) + { + var segments = path.Split('/'); + string search = ""; + foreach (var segment in segments) + { + search = search.Length == 0 ? segment : search + "/" + segment; + if (search == path) + { + _toExclude.Add(search); + break; + } + if (!included.Contains(search)) + { + _toExclude.Add(search); + break; + } + } + } + } + + private static void AddPathAndParents(HashSet included, string? path) + { + while (!string.IsNullOrEmpty(path) && !included.Contains(path)) + { + Preconditions.CheckArgument(!Path.IsPathRooted(path)); + included.Add(path); + path = Parent(path); + } + } + + /// + /// Finds and records the path of all submodules. This should be called when they are not staged + /// for deletion. + /// + internal void FindSubmodules(Console console) + { + _addBackSubmodules = new List(); + + string submoduleStatus = _repo.SimpleCommand("submodule", "status").GetStdout(); + foreach (var line in submoduleStatus.Split('\n')) + { + if (line.Length == 0) + { + continue; + } + string submoduleName = SubmoduleStatusPrefix.Replace(line, "", 1); + if (submoduleName == line) + { + console.Warn("Cannot parse line from 'git submodule status': " + line); + continue; + } + if (!_pathMatcher.Matches(Path.Combine(_workTree, submoduleName))) + { + _addBackSubmodules.Add(submoduleName); + } + } + } + + /// Adds all the excluded files and submodules. + internal void Add() + { + int size = 0; + var current = new List(); + foreach (var path in _toExclude) + { + current.Add(path); + size += path.Length; + // Split the executions in chunks of 6K. 8K triggers arg max in some systems. + if (size > 6 * 1024) + { + _repo.Add().Force().Files(current).Run(); + current = new List(); + size = 0; + } + } + if (current.Count != 0) + { + _repo.Add().Force().Files(current).Run(); + } + + foreach (var addBackSubmodule in _addBackSubmodules ?? new List()) + { + _repo.SimpleCommand("reset", "--", "--quiet", addBackSubmodule); + _repo.Add().Force().Files(new[] { addBackSubmodule }).Run(); + } + } + + private IEnumerable EnumerateRelativeFiles(string workdir) + { + if (!Directory.Exists(workdir)) + { + yield break; + } + foreach (var file in Directory.EnumerateFiles( + workdir, "*", SearchOption.AllDirectories)) + { + yield return Normalize(Path.GetRelativePath(workdir, file)); + } + } + + private static string Normalize(string path) => path.Replace('\\', '/'); + + private static string? Parent(string path) + { + int idx = path.LastIndexOf('/'); + return idx < 0 ? null : path.Substring(0, idx); + } + + private static bool IsHidden(string relativePath) + { + int idx = relativePath.LastIndexOf('/'); + string name = idx < 0 ? relativePath : relativePath.Substring(idx + 1); + return name.StartsWith('.'); + } +} diff --git a/src/Copybara.Core/Git/CannotIntegrateException.cs b/src/Copybara.Core/Git/CannotIntegrateException.cs new file mode 100644 index 000000000..3205966ac --- /dev/null +++ b/src/Copybara.Core/Git/CannotIntegrateException.cs @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Exceptions; + +namespace Copybara.Git; + +/// +/// Thrown if git destination integrate failed for some reason. Port of +/// com.google.copybara.git.CannotIntegrateException. +/// +public class CannotIntegrateException : ValidationException +{ + public CannotIntegrateException(string message) + : base(message) + { + } + + public CannotIntegrateException(string message, Exception? cause) + : base(message, cause) + { + } +} diff --git a/src/Copybara.Core/Git/ChangeReader.cs b/src/Copybara.Core/Git/ChangeReader.cs new file mode 100644 index 000000000..1fe0d6548 --- /dev/null +++ b/src/Copybara.Core/Git/ChangeReader.cs @@ -0,0 +1,345 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Revision; +using Copybara.Util; +using Console = Copybara.Util.Console.Console; +using AuthoringT = Copybara.Authoring.Authoring; +using Author = Copybara.Authoring.Author; + +namespace Copybara.Git; + +/// +/// Utility class to introspect the log of a Git repository. Port of +/// com.google.copybara.git.ChangeReader. +/// +internal sealed class ChangeReader +{ + internal const string BranchCommitLogHeading = "-- Branch commit log --"; + + private readonly AuthoringT? _authoring; + private readonly GitRepository _repository; + private readonly int _limit; + private readonly ImmutableArray _roots; + private readonly bool _includeBranchCommitLogs; + private readonly string? _url; + private readonly bool _firstParent; + private readonly bool _partialFetch; + private readonly bool _topoOrder; + private readonly int _skip; + private readonly int _batchSize; + private readonly string? _grepString; + + private ChangeReader( + AuthoringT? authoring, + GitRepository repository, + int limit, + IEnumerable roots, + bool includeBranchCommitLogs, + string? url, + bool firstParent, + bool partialFetch, + bool topoOrder, + int skip, + int batchSize, + string? grepString) + { + _authoring = authoring; + _repository = Preconditions.CheckNotNull(repository, "repository"); + _limit = limit; + _roots = roots.ToImmutableArray(); + _includeBranchCommitLogs = includeBranchCommitLogs; + _url = url; + _firstParent = firstParent; + _partialFetch = partialFetch; + _topoOrder = topoOrder; + _skip = skip; + _batchSize = batchSize; + _grepString = grepString; + } + + internal IReadOnlyList> Run(GitRevision rev) => + Run( + null, + rev, + historyIsNonLinear: false, + ImmutableDictionary>.Empty); + + /// + /// Computes a list of changes from the refExpression and propagates labels contained in + /// onto the resulting change list. + /// + internal IReadOnlyList> Run( + GitRevision? fromRev, + GitRevision toRev, + bool historyIsNonLinear, + IReadOnlyDictionary> labels) + { + string refExpression = + fromRev == null || historyIsNonLinear + ? toRev.GetHash() + : fromRev.GetHash() + ".." + toRev.GetHash(); + GitRepository.LogCmd logCmd = + _repository.Log(refExpression).FirstParent(_firstParent).TopoOrder(_topoOrder); + if (_limit != -1) + { + logCmd = logCmd.WithLimit(_limit); + } + if (_skip > 0) + { + logCmd = logCmd.WithSkip(_skip); + } + if (_batchSize > 0) + { + logCmd = logCmd.WithBatchSize(_batchSize); + } + if (_grepString != null) + { + logCmd = logCmd.Grep(_grepString); + } + if (_partialFetch && _roots.Contains("")) + { + throw new ValidationException( + "Config error: partial_fetch feature is not compatible with fetching the whole" + + " repo."); + } + if (_partialFetch) + { + logCmd = logCmd.WithPaths(_roots); + } + + // Log command does not filter by roots here because of how git log works. Some commits (e.g. + // fake merges) might not include the files in the log, and filtering here would return + // incorrect results. We do filter later on the changes to match the actual glob. + return ParseChanges( + logCmd.IncludeFiles(true).IncludeMergeDiff(true).Run(), labels, toRev); + } + + private string BranchCommitLog(GitRevision @ref, IReadOnlyList parents) + { + if (parents.Count <= 1) + { + // Not a merge commit, so don't bother showing full log of branch commits. + return ""; + } + if (!_includeBranchCommitLogs) + { + return ""; + } + + IReadOnlyList entries = + _repository + .Log(parents[0].GetHash() + ".." + @ref.GetHash()) + .WithPaths(Glob.IsEmptyRoot(_roots) ? ImmutableArray.Empty : _roots) + .FirstParent(false) + .Run(); + + if (entries.Count == 0) + { + return ""; + } + // Remove the merge commit. Since we already have that in the body. + entries = entries.Skip(1).ToList(); + + var sb = new StringBuilder("\n").Append(BranchCommitLogHeading).Append('\n'); + bool first = true; + foreach (var e in entries) + { + if (!first) + { + sb.Append('\n'); + } + sb.Append("commit ") + .Append(e.Commit.GetHash()) + .Append('\n') + .Append("Author: ") + .Append(FilterAuthor(e.Author)) + .Append('\n') + .Append("Date: ") + .Append(e.AuthorDate) + .Append('\n') + .Append('\n') + .Append(" ") + .Append((e.Body ?? "").Replace("\n", " \n")); + first = false; + } + return sb.ToString(); + } + + private IReadOnlyList> ParseChanges( + IReadOnlyList logEntries, + IReadOnlyDictionary> labels, + GitRevision toRev) + { + var result = new List>(); + GitRevision? last = null; + foreach (var e in logEntries) + { + // Keep the first commit if repeated (merge commits). + if (last != null && last.Equals(e.Commit)) + { + continue; + } + last = e.Commit; + ImmutableListMultimap labelsToCopy = + labels.TryGetValue(e.Commit.GetHash(), out var found) + ? found + : ImmutableListMultimap.Empty; + // Carry over the context reference to the corresponding change in the list. + if (last.GetHash() == toRev.GetHash() && toRev.ContextReference() != null) + { + last = last.WithContextReference(toRev.ContextReference()!); + } + result.Add( + new Change( + (_url != null ? last.WithUrl(_url) : last).WithLabels(labelsToCopy), + FilterAuthor(e.Author), + (e.Body ?? "") + BranchCommitLog(last, e.Parents), + e.AuthorDate, + ChangeMessage.ParseAllAsLabels(e.Body ?? "").LabelsAsMultimap(), + e.Files, + e.Parents.Count > 1, + e.Parents.ToImmutableArray())); + } + result.Reverse(); + return result; + } + + private Author FilterAuthor(Author author) => + _authoring == null || _authoring.UseAuthor(author.Email) + ? author + : _authoring.GetDefaultAuthor(); + + /// Builder for . + internal sealed class Builder + { + private AuthoringT? _authoring; + private readonly GitRepository _repository; + private int _limit = -1; + private ImmutableArray _roots = ImmutableArray.Create(""); + private bool _includeBranchCommitLogs; + private string? _url; + private bool _firstParent; + private bool _topoOrder; + private bool _partialFetch; + private int _skip; + private int _batchSize; + private string? _grepString; + + internal static Builder ForDestination(GitRepository repository, Console console) => + new(repository, console); + + internal static Builder ForOrigin( + AuthoringT authoring, GitRepository repository, Console console) => + new Builder(repository, console).SetAuthoring(authoring); + + private Builder(GitRepository repository, Console console) + { + _repository = Preconditions.CheckNotNull(repository, "repository"); + Preconditions.CheckNotNull(console, "console"); + } + + internal Builder SetLimit(int limit) + { + Preconditions.CheckArgument(limit > 0); + _limit = limit; + return this; + } + + internal Builder SetSkip(int skip) + { + Preconditions.CheckArgument(skip >= 0); + _skip = skip; + return this; + } + + internal Builder SetBatchSize(int batchSize) + { + Preconditions.CheckArgument(batchSize >= 0); + _batchSize = batchSize; + return this; + } + + private Builder SetAuthoring(AuthoringT authoring) + { + _authoring = Preconditions.CheckNotNull(authoring, "authoring"); + return this; + } + + internal Builder SetPartialFetch(bool partialFetch) + { + _partialFetch = partialFetch; + return this; + } + + internal Builder SetFirstParent(bool firstParent) + { + _firstParent = firstParent; + return this; + } + + internal Builder SetTopoOrder(bool topoOrder) + { + _topoOrder = topoOrder; + return this; + } + + internal Builder SetIncludeBranchCommitLogs(bool includeBranchCommitLogs) + { + _includeBranchCommitLogs = includeBranchCommitLogs; + return this; + } + + internal Builder SetUrl(string? url) + { + _url = url; + return this; + } + + /// Only return commits that match the given paths in the Git log command. + internal Builder SetRoots(IEnumerable roots) + { + _roots = roots.ToImmutableArray(); + return this; + } + + /// Grep for the given pattern in the Git log command. + internal Builder Grep(string grepString) + { + _grepString = grepString; + return this; + } + + internal ChangeReader Build() => + new( + _authoring, + _repository, + _limit, + _roots, + _includeBranchCommitLogs, + _url, + _firstParent, + _partialFetch, + _topoOrder, + _skip, + _batchSize, + _grepString); + } +} diff --git a/src/Copybara.Core/Git/CredentialFileHandler.cs b/src/Copybara.Core/Git/CredentialFileHandler.cs new file mode 100644 index 000000000..f806da811 --- /dev/null +++ b/src/Copybara.Core/Git/CredentialFileHandler.cs @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2024 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text; +using System.Text.RegularExpressions; +using Copybara.Common; +using Copybara.Credentials; +using Copybara.Exceptions; + +namespace Copybara.Git; + +/// +/// Holder to handle https access tokens for Git Repos. Port of +/// com.google.copybara.git.CredentialFileHandler. +/// +public class CredentialFileHandler +{ + private static readonly object FileLock = new(); + + private readonly string _scheme; + private readonly string _host; + private readonly string _path; + private readonly CredentialIssuer _username; + private readonly CredentialIssuer _password; + private Credential? _currentPassword; + private Credential? _currentUsername; + private readonly object _sync = new(); + + private readonly bool _enabled; + + public CredentialFileHandler( + string scheme, + string host, + string path, + CredentialIssuer username, + CredentialIssuer password, + bool enabled) + { + _scheme = Preconditions.CheckNotNull(scheme); + _host = Preconditions.CheckNotNull(host); + _path = Preconditions.CheckNotNull(path); + _username = Preconditions.CheckNotNull(username); + _password = Preconditions.CheckNotNull(password); + _enabled = enabled; + } + + public CredentialFileHandler( + string host, string path, CredentialIssuer username, CredentialIssuer password, bool enable) + : this("https", host, path, username, password, enable) + { + } + + public CredentialFileHandler( + string host, string path, CredentialIssuer username, CredentialIssuer password) + : this("https", host, path, username, password, true) + { + } + + public CredentialFileHandler( + string scheme, string host, string path, CredentialIssuer username, CredentialIssuer password) + : this(scheme, host, path, username, password, true) + { + } + + /// Obtain a token for the username field from the username Issuer. + public string GetUsername() => GetUsernameCred().ProvideSecret(); + + /// Obtain a token for the password field from the password Issuer. + public string GetPassword() => GetPasswordCred().ProvideSecret(); + + private Credential GetPasswordCred() + { + lock (_sync) + { + if (_currentPassword == null || !_currentPassword.Valid()) + { + _currentPassword = _password.Issue(); + } + return _currentPassword; + } + } + + private Credential GetUsernameCred() + { + lock (_sync) + { + if (_currentUsername == null || !_currentUsername.Valid()) + { + _currentUsername = _username.Issue(); + } + return _currentUsername; + } + } + + public void Install(GitRepository repo, string credentialHelper) + { + if (!_enabled) + { + return; + } + repo.ReplaceLocalConfigField("credential", "useHttpPath", "true"); + WriteTokenToCredFile(credentialHelper); + repo.WithCredentialHelper("store --file=" + credentialHelper); + } + + /// + /// Writes an entry for the token into the given creds file. If the token has expired, calling + /// this again will update the token. + /// + public void WriteTokenToCredFile(string file) + { + lock (FileLock) + { + try + { + var lines = new List(); + if (File.Exists(file)) + { + lines.AddRange(File.ReadAllLines(file)); + } + string entry; + Regex pattern; + try + { + entry = GetCredentialEntry(GetPasswordCred().ProvideSecret()); + pattern = new Regex( + "^" + + Regex.Escape(GetCredentialEntry("PASSWORD_PLACEHOLDER")) + .Replace("PASSWORD_PLACEHOLDER", "[^@]+") + + "$"); + } + catch (Exception e) + when (e is CredentialRetrievalException or CredentialIssuingException) + { + throw new RepoException("Issue minting token", e); + } + bool missing = true; + for (int i = 0; i < lines.Count; i++) + { + string line = lines[i]; + if (line == entry) + { + return; + } + if (pattern.IsMatch(line)) + { + lines[i] = entry; + missing = false; + } + } + if (missing) + { + lines.Add(entry); + } + File.WriteAllText( + file, + string.Join("\n", lines.Where(s => !string.IsNullOrEmpty(s))) + "\n"); + } + catch (IOException e) + { + throw new RepoException($"Error writing access token for {_host}/{_path}", e); + } + } + } + + private string GetCredentialEntry(string pw) => + $"{_scheme}://{Uri.EscapeDataString(GetUsername())}:{Uri.EscapeDataString(pw)}@{_host}/{_path}"; + + /// + /// Helper to print a cred files without exposing tokens. Do not use this output for anything but + /// debugging. + /// + public string GetScrubbedFileContentForDebug(string file) + { + if (!File.Exists(file)) + { + return ""; + } + try + { + return ScrubCredential(File.ReadAllText(file)); + } + catch (IOException e) + { + return ""; + } + } + + private static string ScrubCredential(string line) => + Regex.Replace( + line, "([^:\\n]+://[^:\\n]+):[^@\\n]+(@[^\\n]*)", "$1:$2"); + + public override string ToString() => + $"CredentialFileHandler{{host={_host}, path={_path}, password={_password.Describe()}," + + $" username={_username.Describe()}}}"; + + public IReadOnlyList> DescribeCredentials() => + ImmutableArray.Create(_username.Describe(), _password.Describe()); +} diff --git a/src/Copybara.Core/Git/EventTrigger.cs b/src/Copybara.Core/Git/EventTrigger.cs new file mode 100644 index 000000000..78b5bf81c --- /dev/null +++ b/src/Copybara.Core/Git/EventTrigger.cs @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2021 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Git.GitHub.Api; + +namespace Copybara.Git; + +/// +/// A simple pair to express GitHub Events with arbitrary subtypes (Status, CheckRun). Port of +/// com.google.copybara.git.EventTrigger. +/// +public sealed class EventTrigger +{ + private readonly GitHubEventType _type; + private readonly ImmutableHashSet _subtypes; + + private EventTrigger(GitHubEventType type, ImmutableHashSet subtypes) + { + _type = type; + _subtypes = subtypes; + } + + public GitHubEventType Type() => _type; + + public IReadOnlySet Subtypes() => _subtypes; + + public static EventTrigger Create(GitHubEventType type, IEnumerable subtypes) => + new(type, subtypes.ToImmutableHashSet()); + + public override bool Equals(object? o) => + o is EventTrigger other && _type == other._type && _subtypes.SetEquals(other._subtypes); + + public override int GetHashCode() + { + var hash = new HashCode(); + hash.Add(_type); + foreach (var s in _subtypes.OrderBy(x => x, StringComparer.Ordinal)) + { + hash.Add(s); + } + return hash.ToHashCode(); + } + + public override string ToString() => + $"EventTrigger{{type={_type}, subtypes=[{string.Join(", ", _subtypes)}]}}"; +} diff --git a/src/Copybara.Core/Git/FetchResult.cs b/src/Copybara.Core/Git/FetchResult.cs new file mode 100644 index 000000000..9540926d5 --- /dev/null +++ b/src/Copybara.Core/Git/FetchResult.cs @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; + +namespace Copybara.Git; + +/// +/// The result of executing a git fetch command. Port of +/// com.google.copybara.git.FetchResult. +/// +public sealed class FetchResult +{ + private readonly ImmutableDictionary _deleted; + private readonly ImmutableDictionary _inserted; + private readonly ImmutableDictionary _updated; + + public FetchResult( + IReadOnlyDictionary before, + IReadOnlyDictionary after) + { + var deleted = ImmutableDictionary.CreateBuilder(); + var inserted = ImmutableDictionary.CreateBuilder(); + var updated = ImmutableDictionary.CreateBuilder(); + + foreach (var entry in before) + { + if (!after.TryGetValue(entry.Key, out var afterVal)) + { + deleted[entry.Key] = entry.Value; + } + else if (!Equals(afterVal, entry.Value)) + { + updated[entry.Key] = new RefUpdate(entry.Value, afterVal); + } + } + foreach (var entry in after) + { + if (!before.ContainsKey(entry.Key)) + { + inserted[entry.Key] = entry.Value; + } + } + + _deleted = deleted.ToImmutable(); + _inserted = inserted.ToImmutable(); + _updated = updated.ToImmutable(); + } + + public override string ToString() => + $"FetchResult{{deleted={FormatMap(_deleted)}, inserted={FormatMap(_inserted)}," + + $" updated={FormatMap(_updated)}}}"; + + private static string FormatMap(ImmutableDictionary map) => + "{" + string.Join(", ", map.Select(e => $"{e.Key}={e.Value}")) + "}"; + + public IReadOnlyDictionary GetDeleted() => _deleted; + + public IReadOnlyDictionary GetInserted() => _inserted; + + public IReadOnlyDictionary GetUpdated() => _updated; + + /// A reference update for a fetch command. Contains before and after SHA-1. + public sealed class RefUpdate + { + private readonly GitRevision _before; + private readonly GitRevision _after; + + public RefUpdate(GitRevision before, GitRevision after) + { + _before = before; + _after = after; + } + + public GitRevision GetBefore() => _before; + + public GitRevision GetAfter() => _after; + + public override string ToString() => _before.GetHash() + " -> " + _after.GetHash(); + } +} diff --git a/src/Copybara.Core/Git/FuzzyClosestVersionSelector.cs b/src/Copybara.Core/Git/FuzzyClosestVersionSelector.cs new file mode 100644 index 000000000..7309284af --- /dev/null +++ b/src/Copybara.Core/Git/FuzzyClosestVersionSelector.cs @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Exceptions; +using Copybara.Git.Version; +using Copybara.Go; +using Copybara.Version; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Git; + +/// +/// A VersionSelector that heuristically tries to match a version to a git tag. This is best effort +/// and only recommended for testing. Port of +/// com.google.copybara.git.FuzzyClosestVersionSelector. +/// +public class FuzzyClosestVersionSelector +{ + public string SelectVersion(string? requestedRef, GitRepository repo, string url, Console console) + { + // Move this check where it is used + ValidationException.CheckCondition( + !string.IsNullOrEmpty(requestedRef), + "Fuzzy version finding requires a ref to be explicitly specified"); + + var selector = + new OrderedVersionSelector( + ImmutableArray.Create( + new PseudoVersionSelector(), + new RequestedShaVersionSelector(), + new RequestedExactMatchSelector(), + new CorrectorVersionSelector(console), + new RequestedVersionSelector())); + try + { + return selector.Select( + new RefspecVersionList.TagVersionList(repo, url), requestedRef, console)!; + } + catch (RepoException e) + { + // Technically this could be a real RepoException, but the current interface + // + console.WarnFmt("Unable to obtain tags for {0}. {1}", url, e); + return requestedRef!; + } + } +} diff --git a/src/Copybara.Core/Git/GerritApi/AbandonInput.cs b/src/Copybara.Core/Git/GerritApi/AbandonInput.cs new file mode 100644 index 000000000..300dd4c20 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/AbandonInput.cs @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#abandon-input +/// +/// NotifyInfo (notify_details) not included for now. +/// +public class AbandonInput : IStarlarkPrintableValue +{ + [JsonPropertyName("message")] + public string? Message { get; set; } + + [JsonPropertyName("notify")] + public string? Notify { get; set; } + + private AbandonInput(string? message, NotifyType? notify) + { + Message = message; + Notify = notify?.ToWireValue(); + } + + public static AbandonInput Create(string? message, NotifyType? notify) => + new(message, notify); + + public static AbandonInput CreateWithoutComment() => new(null, null); + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => $"AbandonInput{{message={Message}, notify={Notify}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/AccountInfo.cs b/src/Copybara.Core/Git/GerritApi/AccountInfo.cs new file mode 100644 index 000000000..4366967e1 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/AccountInfo.cs @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Globalization; +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// https://gerrit-review.googlesource.com/Documentation/rest-api-accounts.html#account-info +[StarlarkBuiltin("gerritapi.AccountInfo", Doc = "Gerrit account information.")] +public class AccountInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("_account_id")] + public long AccountId { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("email")] + public string? Email { get; set; } + + [JsonPropertyName("secondary_emails")] + public IReadOnlyList? SecondaryEmails { get; set; } + + [JsonPropertyName("username")] + public string? Username { get; set; } + + public AccountInfo() + { + } + + public AccountInfo(long accountId, string? email) + { + AccountId = accountId; + Email = email; + } + + public long GetAccountId() => AccountId; + + [StarlarkMethod( + "account_id", + Doc = "The numeric ID of the account.", + StructField = true)] + public string GetAccountIdAsString() => AccountId.ToString(CultureInfo.InvariantCulture); + + [StarlarkMethod( + "name", + Doc = + "The full name of the user.\n" + + "Only set if detailed account information is requested.\n" + + "See option DETAILED_ACCOUNTS for change queries\n" + + "and option DETAILS for account queries.", + StructField = true, + AllowReturnNones = true)] + public string? GetName() => Name; + + [StarlarkMethod( + "email", + Doc = + "The email address the user prefers to be contacted through.\n" + + "Only set if detailed account information is requested.\n" + + "See option DETAILED_ACCOUNTS for change queries\n" + + "and options DETAILS and ALL_EMAILS for account queries.", + StructField = true, + AllowReturnNones = true)] + public string? GetEmail() => Email; + + [StarlarkMethod( + "secondary_emails", + Doc = + "A list of the secondary email addresses of the user.\n" + + "Only set for account queries when the ALL_EMAILS option or the suggest " + + "parameter is set.\n" + + "Secondary emails are only included if the calling user has the Modify Account, " + + "and hence is allowed to see secondary emails of other users.", + StructField = true)] + public IReadOnlyList GetSecondaryEmails() => + SecondaryEmails is null ? ImmutableArray.Empty : SecondaryEmails.ToImmutableArray(); + + [StarlarkMethod( + "username", + Doc = + "The username of the user.\n" + + "Only set if detailed account information is requested.\n" + + "See option DETAILED_ACCOUNTS for change queries\n" + + "and option DETAILS for account queries.", + StructField = true, + AllowReturnNones = true)] + public string? GetUsername() => Username; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"AccountInfo{{accountId={AccountId}, name={Name}, email={Email}, " + + $"secondaryEmails={FormatList(SecondaryEmails)}, username={Username}}}"; + + private protected static string FormatList(IEnumerable? items) => + items is null ? "null" : "[" + string.Join(", ", items) + "]"; +} diff --git a/src/Copybara.Core/Git/GerritApi/ActionInfo.cs b/src/Copybara.Core/Git/GerritApi/ActionInfo.cs new file mode 100644 index 000000000..377367751 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ActionInfo.cs @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#action-info +[StarlarkBuiltin("gerritapi.getActionInfo", Doc = "Gerrit actions information.")] +public class ActionInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("method")] + public string? Method { get; set; } + + [JsonPropertyName("label")] + public string? Label { get; set; } + + [JsonPropertyName("title")] + public string? Title { get; set; } + + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } + + public ActionInfo() + { + } + + public ActionInfo(string method, string label, string title, bool enabled) + { + Method = method; + Label = label; + Title = title; + Enabled = enabled; + } + + public string? GetMethod() => Method; + + [StarlarkMethod( + "label", + Doc = "Short title to display to a user describing the action", + StructField = true, + AllowReturnNones = true)] + public string? GetLabel() => Label; + + public string? GetTitle() => Title; + + [StarlarkMethod( + "enabled", + Doc = + "If true the action is permitted at this time and the caller is likely " + + "allowed to execute it.", + StructField = true)] + public bool GetEnabled() => Enabled; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"ActionInfo{{method={Method}, label={Label}, title={Title}, enabled={Enabled}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/ApprovalInfo.cs b/src/Copybara.Core/Git/GerritApi/ApprovalInfo.cs new file mode 100644 index 000000000..37b448cb1 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ApprovalInfo.cs @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; + +namespace Copybara.Git.GerritApi; + +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#approval-info +[StarlarkBuiltin("gerritapi.ApprovalInfo", Doc = "Gerrit approval information.")] +public class ApprovalInfo : AccountInfo +{ + [JsonPropertyName("value")] + public int Value { get; set; } + + [JsonPropertyName("date")] + public string? Date { get; set; } + + public ApprovalInfo() + { + } + + public ApprovalInfo(long accountId, string? email, int value) + : base(accountId, email) + { + Value = value; + } + + [StarlarkMethod( + "value", + Doc = + "The vote that the user has given for the label. If present and zero, the user " + + "is permitted to vote on the label. If absent, the user is not permitted to vote " + + "on that label.", + StructField = true)] + public int GetValue() => Value; + + public DateTimeOffset GetDate() => GerritApiUtil.ParseTimestamp(Date!); + + [StarlarkMethod( + "date", + Doc = "The time and date describing when the approval was made.", + StructField = true, + AllowReturnNones = true)] + public string? GetDateForSkylark() => Date; + + public override string ToString() => + $"ApprovalInfo{{accountId={AccountId}, name={Name}, email={Email}, " + + $"secondaryEmails={FormatList(SecondaryEmails)}, username={Username}, " + + $"value={Value}, date={Date}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/ChangeInfo.cs b/src/Copybara.Core/Git/GerritApi/ChangeInfo.cs new file mode 100644 index 000000000..a011d9605 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ChangeInfo.cs @@ -0,0 +1,286 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Globalization; +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#change-info +[StarlarkBuiltin("gerritapi.ChangeInfo", Doc = "Gerrit change information.")] +public class ChangeInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("triplet_id")] + public string? TripletId { get; set; } + + [JsonPropertyName("project")] + public string? Project { get; set; } + + [JsonPropertyName("branch")] + public string? Branch { get; set; } + + [JsonPropertyName("topic")] + public string? Topic { get; set; } + + [JsonPropertyName("change_id")] + public string? ChangeIdField { get; set; } + + [JsonPropertyName("subject")] + public string? Subject { get; set; } + + [JsonPropertyName("status")] + public string? StatusString { get; set; } + + [JsonPropertyName("created")] + public string? Created { get; set; } + + [JsonPropertyName("updated")] + public string? Updated { get; set; } + + [JsonPropertyName("submitted")] + public string? Submitted { get; set; } + + [JsonPropertyName("submittable")] + public bool Submittable { get; set; } + + [JsonPropertyName("work_in_progress")] + public bool WorkInProgress { get; set; } + + [JsonPropertyName("_number")] + public long Number { get; set; } + + [JsonPropertyName("owner")] + public AccountInfo? Owner { get; set; } + + [JsonPropertyName("submit_requirements")] + public IReadOnlyList? SubmitRequirementsField { get; set; } + + [JsonPropertyName("labels")] + public IReadOnlyDictionary? LabelsField { get; set; } + + [JsonPropertyName("messages")] + public IReadOnlyList? MessagesField { get; set; } + + [JsonPropertyName("current_revision")] + public string? CurrentRevision { get; set; } + + [JsonPropertyName("revisions")] + public IReadOnlyDictionary? AllRevisionsField { get; set; } + + [JsonPropertyName("_more_changes")] + public bool MoreChanges { get; set; } + + [JsonPropertyName("reviewers")] + public IReadOnlyDictionary>? ReviewersField { get; set; } + + [StarlarkMethod( + "id", + Doc = + "The ID of the change in the format \"`~~`\", where " + + "'project', 'branch' and 'Change-Id' are URL encoded. For 'branch' the " + + "refs/heads/ prefix is omitted.", + StructField = true, + AllowReturnNones = true)] + public string? GetId() => Id; + + [StarlarkMethod( + "triplet_id", + Doc = + "The ID of the change in the format \"'~~'\", where 'project'" + + " and 'branch' are URL encoded. For 'branch' the refs/heads/ prefix is omitted.", + StructField = true, + AllowReturnNones = true)] + public string? GetTripletId() => TripletId; + + [StarlarkMethod( + "project", + Doc = "The name of the project.", + StructField = true, + AllowReturnNones = true)] + public string? GetProject() => Project; + + [StarlarkMethod( + "branch", + Doc = "The name of the target branch.\nThe refs/heads/ prefix is omitted.", + StructField = true, + AllowReturnNones = true)] + public string? GetBranch() => Branch; + + [StarlarkMethod( + "topic", + Doc = "The topic to which this change belongs.", + StructField = true, + AllowReturnNones = true)] + public string? GetTopic() => Topic; + + [StarlarkMethod( + "change_id", + Doc = "The Change-Id of the change.", + StructField = true, + AllowReturnNones = true)] + public string? GetChangeId() => ChangeIdField; + + [StarlarkMethod( + "subject", + Doc = "The subject of the change (header line of the commit message).", + StructField = true, + AllowReturnNones = true)] + public string? GetSubject() => Subject; + + public ChangeStatus GetStatus() => Enum.Parse(StatusString!); + + [StarlarkMethod( + "status", + Doc = "The status of the change (NEW, MERGED, ABANDONED).", + StructField = true, + AllowReturnNones = true)] + public string? GetStatusAsString() => StatusString; + + public DateTimeOffset GetCreated() => GerritApiUtil.ParseTimestamp(Created!); + + [StarlarkMethod( + "created", + Doc = "The timestamp of when the change was created.", + StructField = true, + AllowReturnNones = true)] + public string? GetCreatedForSkylark() => Created; + + public DateTimeOffset GetUpdated() => GerritApiUtil.ParseTimestamp(Updated!); + + [StarlarkMethod( + "updated", + Doc = "The timestamp of when the change was last updated.", + StructField = true, + AllowReturnNones = true)] + public string? GetUpdatedForSkylark() => Updated; + + public DateTimeOffset GetSubmitted() => GerritApiUtil.ParseTimestamp(Submitted!); + + [StarlarkMethod( + "submitted", + Doc = "The timestamp of when the change was submitted.", + StructField = true, + AllowReturnNones = true)] + public string? GetSubmittedForSkylark() => Submitted; + + [StarlarkMethod( + "submittable", + Doc = + "Whether the change has been approved by the project submit rules. Only set if " + + "requested via additional field SUBMITTABLE.", + StructField = true)] + public bool IsSubmittable() => Submittable; + + [StarlarkMethod( + "work_in_progress", + Doc = "Whether the change is marked as \"Work in progress\".", + StructField = true)] + public bool IsWorkInProgress() => WorkInProgress; + + public long GetNumber() => Number; + + [StarlarkMethod("number", Doc = "The legacy numeric ID of the change.", StructField = true)] + public string GetNumberAsString() => Number.ToString(CultureInfo.InvariantCulture); + + [StarlarkMethod( + "owner", + Doc = "The owner of the change as an AccountInfo entity.", + StructField = true, + AllowReturnNones = true)] + public AccountInfo? GetOwner() => Owner; + + public IReadOnlyList GetSubmitRequirements() => + SubmitRequirementsField is null + ? ImmutableArray.Empty + : SubmitRequirementsField.ToImmutableArray(); + + [StarlarkMethod( + "submit_requirements", + Doc = "A list of the evaluated submit requirements for the change.", + StructField = true)] + public IReadOnlyList GetSubmitRequirementsForSkylark() => + GetSubmitRequirements(); + + public IReadOnlyDictionary GetLabels() => + LabelsField is null ? ImmutableDictionary.Empty : LabelsField.ToImmutableDictionary(); + + [StarlarkMethod( + "labels", + Doc = + "The labels of the change as a map that maps the label names to LabelInfo entries.\n" + + "Only set if labels or detailed labels are requested.", + StructField = true)] + public IReadOnlyDictionary GetLabelsForSkylark() => GetLabels(); + + public IReadOnlyList GetMessages() => + MessagesField is null ? ImmutableArray.Empty : MessagesField.ToImmutableArray(); + + [StarlarkMethod( + "messages", + Doc = + "Messages associated with the change as a list of ChangeMessageInfo entities.\n" + + "Only set if messages are requested.", + StructField = true)] + public IReadOnlyList GetMessagesForSkylark() => GetMessages(); + + [StarlarkMethod( + "current_revision", + Doc = + "The commit ID of the current patch set of this change.\n" + + "Only set if the current revision is requested or if all revisions are requested.", + StructField = true, + AllowReturnNones = true)] + public string? GetCurrentRevision() => CurrentRevision; + + public IReadOnlyDictionary GetAllRevisions() => + AllRevisionsField is null + ? ImmutableDictionary.Empty + : AllRevisionsField.ToImmutableDictionary(); + + [StarlarkMethod( + "revisions", + Doc = + "All patch sets of this change as a map that maps the commit ID of the patch set to a " + + "RevisionInfo entity.\n" + + "Only set if the current revision is requested (in which case it will only contain " + + "a key for the current revision) or if all revisions are requested.", + StructField = true)] + public IReadOnlyDictionary GetAllRevisionsForSkylark() => GetAllRevisions(); + + public IReadOnlyDictionary> GetReviewers() => + ReviewersField is null + ? ImmutableDictionary>.Empty + : ReviewersField.ToImmutableDictionary(); + + public bool IsMoreChanges() => MoreChanges; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"ChangeInfo{{id={Id}, project={Project}, branch={Branch}, topic={Topic}, " + + $"changeId={ChangeIdField}, subject={Subject}, status={StatusString}, created={Created}, " + + $"updated={Updated}, submitted={Submitted}, submittable={Submittable}, " + + $"work_in_progress={WorkInProgress}, number={Number}, owner={Owner}, " + + $"submitRequirements={SubmitRequirementsField}, labels={LabelsField}, " + + $"messages={MessagesField}, currentRevision={CurrentRevision}, " + + $"allRevisions={AllRevisionsField}, moreChanges={MoreChanges}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/ChangeMessageInfo.cs b/src/Copybara.Core/Git/GerritApi/ChangeMessageInfo.cs new file mode 100644 index 000000000..7358ec336 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ChangeMessageInfo.cs @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#change-message-info +/// +[StarlarkBuiltin("gerritapi.ChangeMessageInfo", Doc = "Gerrit change message information.")] +public class ChangeMessageInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("author")] + public AccountInfo? Author { get; set; } + + [JsonPropertyName("real_author")] + public AccountInfo? RealAuthor { get; set; } + + [JsonPropertyName("date")] + public string? Date { get; set; } + + [JsonPropertyName("message")] + public string? Message { get; set; } + + [JsonPropertyName("tag")] + public string? Tag { get; set; } + + [JsonPropertyName("_revision_number")] + public int RevisionNumber { get; set; } + + [StarlarkMethod( + "id", + Doc = "The ID of the message.", + StructField = true, + AllowReturnNones = true)] + public string? GetId() => Id; + + [StarlarkMethod( + "author", + Doc = + "Author of the message as an AccountInfo entity.\n" + + "Unset if written by the Gerrit system.", + StructField = true, + AllowReturnNones = true)] + public AccountInfo? GetAuthor() => Author; + + [StarlarkMethod( + "real_author", + Doc = + "Real author of the message as an AccountInfo entity.\n" + + "Set if the message was posted on behalf of another user.", + StructField = true, + AllowReturnNones = true)] + public AccountInfo? GetRealAuthor() => RealAuthor; + + public DateTimeOffset GetDate() => GerritApiUtil.ParseTimestamp(Date!); + + [StarlarkMethod( + "date", + Doc = "The timestamp of when this identity was constructed.", + StructField = true, + AllowReturnNones = true)] + public string? GetDateForSkylark() => Date; + + [StarlarkMethod( + "message", + Doc = "The text left by the user.", + StructField = true, + AllowReturnNones = true)] + public string? GetMessage() => Message; + + [StarlarkMethod( + "tag", + Doc = + "Value of the tag field from ReviewInput set while posting the review. " + + "NOTE: To apply different tags on on different votes/comments multiple " + + "invocations of the REST call are required.", + StructField = true, + AllowReturnNones = true)] + public string? GetTag() => Tag; + + [StarlarkMethod( + "revision_number", + Doc = "Which patchset (if any) generated this message.", + StructField = true)] + public int GetRevisionNumber() => RevisionNumber; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"ChangeMessageInfo{{id={Id}, author={Author}, realAuthor={RealAuthor}, date={Date}, " + + $"message={Message}, tag={Tag}, revisionNumber={RevisionNumber}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/ChangeStatus.cs b/src/Copybara.Core/Git/GerritApi/ChangeStatus.cs new file mode 100644 index 000000000..2363ca1fc --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ChangeStatus.cs @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Git.GerritApi; + +/// +/// Change status for Changes as defined in +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#change-info +/// +public enum ChangeStatus +{ + NEW, + MERGED, + ABANDONED, +} diff --git a/src/Copybara.Core/Git/GerritApi/ChangesQuery.cs b/src/Copybara.Core/Git/GerritApi/ChangesQuery.cs new file mode 100644 index 000000000..c708f3e71 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ChangesQuery.cs @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Copybara.Common; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// +/// An object that represents the input parameters for a changes query: +/// +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes +/// +[Starlark.Annot.StarlarkBuiltin( + "gerritapi.ChangesQuery", + Doc = + "Input for listing Gerrit changes. See " + + "https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes")] +public class ChangesQuery : IStarlarkPrintableValue +{ + private readonly string _query; + private readonly IReadOnlySet _include; + private readonly int? _limit; + private readonly int? _start; + + public ChangesQuery(string query) + { + _query = query; + _include = ImmutableHashSet.Empty; + _limit = null; + _start = null; + } + + private ChangesQuery(string query, IReadOnlySet include, int? limit, int? start) + { + _query = Preconditions.CheckNotNull(query); + _include = Preconditions.CheckNotNull(include); + _limit = limit; + _start = start; + } + + public ChangesQuery WithStart(int start) => new(_query, _include, _limit, start); + + public ChangesQuery WithLimit(int limit) => new(_query, _include, limit, _start); + + public ChangesQuery WithInclude(IEnumerable include) => + new(_query, include.ToImmutableHashSet(), _limit, _start); + + public string AsUrlParams() + { + var sb = new StringBuilder("q=").Append(Escape(_query)); + foreach (var includeResult in _include) + { + sb.Append("&o=").Append(includeResult); + } + + if (_limit != null) + { + sb.Append("&n=").Append(_limit.Value.ToString(CultureInfo.InvariantCulture)); + } + + if (_start != null) + { + sb.Append("&S=").Append(_start.Value.ToString(CultureInfo.InvariantCulture)); + } + + return sb.ToString(); + } + + private static string Escape(string query) => Uri.EscapeDataString(query); + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"ChangesQuery{{query={_query}, include=[{string.Join(", ", _include)}], " + + $"limit={_limit?.ToString(CultureInfo.InvariantCulture) ?? "null"}, " + + $"start={_start?.ToString(CultureInfo.InvariantCulture) ?? "null"}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/CommitInfo.cs b/src/Copybara.Core/Git/GerritApi/CommitInfo.cs new file mode 100644 index 000000000..5c9589a5a --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/CommitInfo.cs @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#commit-info +[StarlarkBuiltin("gerritapi.CommitInfo", Doc = "Gerrit commit information.")] +public class CommitInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("commit")] + public string? Commit { get; set; } + + [JsonPropertyName("parents")] + public IReadOnlyList? Parents { get; set; } + + [JsonPropertyName("author")] + public GitPersonInfo? Author { get; set; } + + [JsonPropertyName("committer")] + public GitPersonInfo? Committer { get; set; } + + [JsonPropertyName("subject")] + public string? Subject { get; set; } + + [JsonPropertyName("message")] + public string? Message { get; set; } + + [StarlarkMethod( + "commit", + Doc = + "The commit ID. Not set if included in a RevisionInfo entity that is contained " + + "in a map which has the commit ID as key.", + StructField = true, + AllowReturnNones = true)] + public string? GetCommit() => Commit; + + public IReadOnlyList GetParents() => + Parents is null ? ImmutableArray.Empty : Parents.ToImmutableArray(); + + [StarlarkMethod( + "parents", + Doc = + "The parent commits of this commit as a list of CommitInfo entities. " + + "In each parent only the commit and subject fields are populated.", + StructField = true)] + public IReadOnlyList GetMessagesForSkylark() => GetParents(); + + [StarlarkMethod( + "author", + Doc = "The author of the commit as a GitPersonInfo entity.", + StructField = true, + AllowReturnNones = true)] + public GitPersonInfo? GetAuthor() => Author; + + [StarlarkMethod( + "committer", + Doc = "The committer of the commit as a GitPersonInfo entity.", + StructField = true, + AllowReturnNones = true)] + public GitPersonInfo? GetCommitter() => Committer; + + [StarlarkMethod( + "subject", + Doc = "The subject of the commit (header line of the commit message).", + StructField = true, + AllowReturnNones = true)] + public string? GetSubject() => Subject; + + [StarlarkMethod( + "message", + Doc = "The commit message.", + StructField = true, + AllowReturnNones = true)] + public string? GetMessage() => Message; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"CommitInfo{{commit={Commit}, parents={Parents}, author={Author}, " + + $"committer={Committer}, subject={Subject}, message={Message}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/DeleteReviewerInput.cs b/src/Copybara.Core/Git/GerritApi/DeleteReviewerInput.cs new file mode 100644 index 000000000..c5154cbd5 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/DeleteReviewerInput.cs @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace Copybara.Git.GerritApi; + +/// +/// See +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#delete-reviewer-input +/// +/// NotifyInfo (notify_details) not included for now. +/// +public class DeleteReviewerInput +{ + [JsonPropertyName("notify")] + public string? Notify { get; set; } + + public DeleteReviewerInput(NotifyType? notify) + { + Notify = notify?.ToWireValue(); + } +} diff --git a/src/Copybara.Core/Git/GerritApi/DeleteVoteInput.cs b/src/Copybara.Core/Git/GerritApi/DeleteVoteInput.cs new file mode 100644 index 000000000..015fe3856 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/DeleteVoteInput.cs @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace Copybara.Git.GerritApi; + +/// +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#delete-vote-input +/// request json. +/// NotifyInfo (notify_details) not included for now. +/// label not included for now since it matches the label in the URL. +/// +public class DeleteVoteInput +{ + [JsonPropertyName("notify")] + public string? Notify { get; set; } + + public DeleteVoteInput(NotifyType? notify) + { + Notify = notify?.ToWireValue(); + } +} diff --git a/src/Copybara.Core/Git/GerritApi/Empty.cs b/src/Copybara.Core/Git/GerritApi/Empty.cs new file mode 100644 index 000000000..897071be8 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/Empty.cs @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Git.GerritApi; + +/// +/// An empty object for operations that return nothing or when we want to ignore the result. +/// +public class Empty +{ +} diff --git a/src/Copybara.Core/Git/GerritApi/FetchInfo.cs b/src/Copybara.Core/Git/GerritApi/FetchInfo.cs new file mode 100644 index 000000000..7c82ea279 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/FetchInfo.cs @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#fetch-info +public class FetchInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("url")] + public string? Url { get; set; } + + [JsonPropertyName("ref")] + public string? Ref { get; set; } + + public FetchInfo() + { + } + + public FetchInfo(string url, string @ref) + { + Url = url; + Ref = @ref; + } + + public string? GetUrl() => Url; + + public string? GetRef() => Ref; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => $"FetchInfo{{url={Url}, ref={Ref}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/GerritApi.cs b/src/Copybara.Core/Git/GerritApi/GerritApi.cs new file mode 100644 index 000000000..09586e642 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/GerritApi.cs @@ -0,0 +1,250 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Copybara.Exceptions; +using ProfilerType = Copybara.Profiler.Profiler; + +namespace Copybara.Git.GerritApi; + +/// +/// A mini API for getting and updating Gerrit projects through the Gerrit REST API. Port of +/// com.google.copybara.git.gerritapi.GerritApi. +/// +public class GerritApi +{ + protected readonly IGerritApiTransport Transport; + protected readonly ProfilerType ProfilerInstance; + + public GerritApi(IGerritApiTransport transport, ProfilerType profiler) + { + Transport = Preconditions.CheckNotNull(transport); + ProfilerInstance = Preconditions.CheckNotNull(profiler); + } + + public async Task> GetChangesAsync(ChangesQuery query) + { + using (ProfilerInstance.Start("gerrit_get_changes")) + { + var result = await Transport + .GetAsync>("/changes/?" + query.AsUrlParams()) + .ConfigureAwait(false); + return (result ?? new List()).ToImmutableArray(); + } + } + + public async Task GetChangeAsync(string changeId, GetChangeInput input) + { + using (ProfilerInstance.Start("gerrit_get_change")) + { + return (await Transport + .GetAsync("/changes/" + changeId + "?" + input.AsUrlParams()) + .ConfigureAwait(false))!; + } + } + + public async Task GetChangeDetailAsync(string changeId, GetChangeInput input) + { + using (ProfilerInstance.Start("gerrit_get_change_detail")) + { + return (await Transport + .GetAsync("/changes/" + changeId + "/detail?" + input.AsUrlParams()) + .ConfigureAwait(false))!; + } + } + + public async Task AbandonChangeAsync(string changeId, AbandonInput abandonInput) + { + using (ProfilerInstance.Start("gerrit_abandon_change")) + { + return (await Transport + .PostAsync("/changes/" + changeId + "/abandon", abandonInput) + .ConfigureAwait(false))!; + } + } + + public async Task RestoreChangeAsync(string changeId, RestoreInput restoreInput) + { + using (ProfilerInstance.Start("gerrit_restore_change")) + { + return (await Transport + .PostAsync("/changes/" + changeId + "/restore", restoreInput) + .ConfigureAwait(false))!; + } + } + + public async Task CheckSubmitRequirementAsync( + string changeId, SubmitRequirementInput submitRequirementInput) + { + using (ProfilerInstance.Start("gerrit_check_submit_requirement")) + { + return (await Transport + .PostAsync( + "/changes/" + changeId + "/check.submit_requirement", submitRequirementInput) + .ConfigureAwait(false))!; + } + } + + /// + /// Look for a Gerrit project using its ID. The ID differs from the name in that certain + /// characters are escaped. E.g. plugins%2Freplication vs plugins/replication. + /// + /// a ProjectInfo if project is found, otherwise null. + public async Task GetProjectByIdAsync(string id) + { + using (ProfilerInstance.Start("gerrit_list_projects")) + { + try + { + return await Transport + .GetAsync("/projects/" + id) + .ConfigureAwait(false); + } + catch (GerritApiException e) + { + if (e.GetResponseCode() == GerritApiException.ResponseCodeValue.NOT_FOUND) + { + return null; + } + + throw; + } + } + } + + public async Task CreateProjectAsync(string project) + { + using (ProfilerInstance.Start("gerrit_create_project")) + { + return (await Transport + .PutAsync("/projects/" + Escape(project), new ProjectInput()) + .ConfigureAwait(false))!; + } + } + + private static string Escape(string project) + { + // Gerrit does a good validation in the server side, but we do some basic checks + ValidationException.CheckCondition( + !project.Contains(' '), "Invalid project name, has spaces: '%s'", project); + return project.Replace("/", "%2F"); + } + + public async Task GetAccessInfoAsync(string project) + { + using (ProfilerInstance.Start("gerrit_access")) + { + return (await Transport + .GetAsync("/projects/" + project + "/access") + .ConfigureAwait(false))!; + } + } + + public async Task SetReviewAsync( + string changeId, string revisionId, SetReviewInput setReviewInput) + { + using (ProfilerInstance.Start("gerrit_set_review")) + { + return (await Transport + .PostAsync( + "/changes/" + changeId + "/revisions/" + revisionId + "/review", setReviewInput) + .ConfigureAwait(false))!; + } + } + + public async Task DeleteReviewerAsync( + string changeId, long accountId, DeleteReviewerInput deleteReviewerInput) + { + using (ProfilerInstance.Start("gerrit_delete_reviewer_by_account_id")) + { + await Transport + .PostAsync( + "/changes/" + changeId + "/reviewers/" + accountId + "/delete", + deleteReviewerInput) + .ConfigureAwait(false); + } + } + + public async Task DeleteReviewerAsync( + string changeId, string email, DeleteReviewerInput deleteReviewerInput) + { + using (ProfilerInstance.Start("gerrit_delete_reviewer_by_email")) + { + await Transport + .PostAsync( + "/changes/" + changeId + "/reviewers/" + email + "/delete", deleteReviewerInput) + .ConfigureAwait(false); + } + } + + public async Task AddReviewerAsync(string changeId, ReviewerInput reviewerInput) + { + using (ProfilerInstance.Start("gerrit_add_reviewer")) + { + await Transport + .PostAsync("/changes/" + changeId + "/reviewers", reviewerInput) + .ConfigureAwait(false); + } + } + + public async Task GetSelfAccountAsync() + { + using (ProfilerInstance.Start("gerrit_get_self")) + { + return (await Transport + .GetAsync("/accounts/self") + .ConfigureAwait(false))!; + } + } + + public async Task> GetActionsAsync( + string changeId, string revision) + { + using (ProfilerInstance.Start("gerrit_get_actions")) + { + var result = await Transport + .GetAsync>( + "/changes/" + changeId + "/revisions/" + revision + "/actions") + .ConfigureAwait(false); + return (result ?? new Dictionary()).ToImmutableDictionary(); + } + } + + public async Task DeleteVoteAsync( + string changeId, string accountId, string labelId, DeleteVoteInput deleteVoteInput) + { + using (ProfilerInstance.Start("gerrit_delete_reviewer_vote")) + { + await Transport + .PostAsync( + "/changes/" + changeId + "/reviewers/" + accountId + "/votes/" + labelId + + "/delete", + deleteVoteInput) + .ConfigureAwait(false); + } + } + + public async Task SubmitChangeAsync(string changeId, SubmitInput submitInput) + { + using (ProfilerInstance.Start("gerrit_submit_change")) + { + return (await Transport + .PostAsync("/changes/" + changeId + "/submit", submitInput) + .ConfigureAwait(false))!; + } + } +} diff --git a/src/Copybara.Core/Git/GerritApi/GerritApiException.cs b/src/Copybara.Core/Git/GerritApi/GerritApiException.cs new file mode 100644 index 000000000..15231b553 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/GerritApiException.cs @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.RegularExpressions; +using Copybara.Exceptions; + +namespace Copybara.Git.GerritApi; + +/// Exception that maps to Gerrit Http error codes. +public class GerritApiException : RepoException +{ + public static readonly Regex ErrorPattern = new( + ".*
(.*)
.*", + RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Multiline); + + private static readonly ImmutableDictionary CodeMap = + Enum.GetValues() + .ToImmutableDictionary(rc => (int)rc); + + private readonly string _baseMessage; + private readonly ResponseCodeValue _responseCode; + private readonly string _gerritResponseMsg; + private readonly string _gerritRequestMsg; + private readonly int _exitCode; + + public GerritApiException( + int exitCode, string message, string gerritResponseMsg, string gerritRequest) + : base(message) + { + _baseMessage = message; + _exitCode = exitCode; + _responseCode = ParseResponseCode(exitCode); + _gerritResponseMsg = gerritResponseMsg; + _gerritRequestMsg = gerritRequest; + } + + public ResponseCodeValue GetResponseCode() => _responseCode; + + public int GetExitCode() => _exitCode; + + private static ResponseCodeValue ParseResponseCode(int code) => + CodeMap.TryGetValue(code, out var rc) ? rc : ResponseCodeValue.UNKNOWN; + + public override string Message => + string.Format( + "{0}: Received error with code {1} from Gerrit: {2}\n\nThe request was:\n\n{3}\n\n" + + "The full response was:\n\n{4}", + _baseMessage, + _exitCode, + ExtractError(), + _gerritRequestMsg, + _gerritResponseMsg); + + private string ExtractError() + { + var matcher = ErrorPattern.Match(_gerritResponseMsg); + if (matcher.Success) + { + return matcher.Groups[1].Value; + } + + return _gerritResponseMsg; + } + + public string GetGerritResponseMsg() => _gerritResponseMsg; + + /// + /// Gerrit known response codes. + /// + /// Note that UNKNOWN will be used for any other not in this list. + /// + /// NOTE(port): named ResponseCodeValue to avoid colliding with the + /// GetResponseCode() accessor within the same type; the Java enum is + /// GerritApiException.ResponseCode. + public enum ResponseCodeValue + { + UNKNOWN = 0, + BAD_REQUEST = 400, + FORBIDDEN = 403, + NOT_FOUND = 404, + METHOD_NOT_ALLOWED = 405, + CONFLICT = 409, + PRECONDITION_FAILED = 412, + UNPROCESSABLE_ENTITY = 422, + } +} diff --git a/src/Copybara.Core/Git/GerritApi/GerritApiTransportImpl.cs b/src/Copybara.Core/Git/GerritApi/GerritApiTransportImpl.cs new file mode 100644 index 000000000..eabc97788 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/GerritApiTransportImpl.cs @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Json; + +namespace Copybara.Git.GerritApi; + +/// +/// Implementation of that uses direct http calls. Port of +/// com.google.copybara.git.gerritapi.GerritApiTransportImpl. +/// +/// +/// NOTE(port): the Java original uses google-http-client + gson. This port uses +/// and , and reuses +/// to strip Gerrit's )]}' XSSI no-execute prefix. Credentials +/// are obtained from the git credential helper via , and +/// applied as HTTP Basic auth, mirroring upstream. +/// +public class GerritApiTransportImpl : IGerritApiTransport +{ + private static readonly TimeSpan Timeout = TimeSpan.FromMinutes(1); + + /// Serialization options mirroring gson: omit nulls, no indentation. + private static readonly JsonSerializerOptions RequestOptions = new() + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly GitRepository _repo; + private readonly Uri _uri; + private readonly HttpClient _httpClient; + + public GerritApiTransportImpl(GitRepository repo, Uri uri, HttpClient httpClient) + { + _repo = repo; + _uri = Preconditions.CheckNotNull(uri); + _httpClient = Preconditions.CheckNotNull(httpClient); + } + + public async Task GetAsync(string path) + { + var userPassword = GetCredentialsIfPresent(_uri.ToString()); + var url = GetUrl(path); + using var request = new HttpRequestMessage(HttpMethod.Get, url); + return await ExecuteAsync(request, userPassword, url).ConfigureAwait(false); + } + + public async Task PostAsync(string path, object request) + { + var userPassword = GetCredentials(_uri.ToString()); + var url = GetUrl(path); + using var httpRequest = new HttpRequestMessage(HttpMethod.Post, url) + { + Content = JsonContent(request), + }; + return await ExecuteAsync(httpRequest, userPassword, url).ConfigureAwait(false); + } + + public async Task PutAsync(string path, object request) + { + var userPassword = GetCredentials(_uri.ToString()); + var url = GetUrl(path); + using var httpRequest = new HttpRequestMessage(HttpMethod.Put, url) + { + Content = JsonContent(request), + }; + return await ExecuteAsync(httpRequest, userPassword, url).ConfigureAwait(false); + } + + public Uri GetUrl(string path) + { + Preconditions.CheckArgument(path.StartsWith('/'), path); + return new Uri(_uri, _uri.AbsolutePath + path); + } + + private static HttpContent JsonContent(object request) + { + string json = JsonSerializer.Serialize(request, request.GetType(), RequestOptions); + return new StringContent(json, Encoding.UTF8, "application/json"); + } + + private async Task ExecuteAsync( + HttpRequestMessage request, GitCredential.UserPassword? userPassword, Uri url) + { + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + if (userPassword != null) + { + var raw = $"{userPassword.GetUsername()}:{userPassword.GetPasswordBeCareful()}"; + var encoded = Convert.ToBase64String(Encoding.UTF8.GetBytes(raw)); + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", encoded); + } + + HttpResponseMessage response; + try + { + using var cts = new CancellationTokenSource(Timeout); + response = await _httpClient.SendAsync(request, cts.Token).ConfigureAwait(false); + } + catch (Exception e) when (e is HttpRequestException or OperationCanceledException) + { + throw new RepoException("Error running Gerrit API operation " + url, e); + } + + using (response) + { + if (!response.IsSuccessStatusCode) + { + string content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new GerritApiException( + (int)response.StatusCode, "Error calling gerrit", content, url.ToString()); + } + + try + { + return await GsonParserUtil + .ParseHttpResponseAsync(response, stripNoExecutePrefix: true) + .ConfigureAwait(false); + } + catch (ArgumentException e) + { + throw new RepoException( + string.Format( + "Cannot parse response as type {0}.\nRequest: {1}\n", typeof(T), url), + e); + } + } + } + + /// + /// Credentials for API should be optional for any read operation (GET). + /// + private GitCredential.UserPassword? GetCredentialsIfPresent(string url) + { + try + { + return GetCredentials(url); + } + catch (ValidationException) + { + return null; + } + } + + /// Gets the credentials from the git credential helper. + /// + /// + private GitCredential.UserPassword GetCredentials(string url) + { + try + { + return _repo.CredentialFill(url); + } + catch (ValidationException e) + { + throw new ValidationException( + $"Cannot get credentials for host {url}, from credentials helper", e); + } + } +} diff --git a/src/Copybara.Core/Git/GerritApi/GerritApiTransportWithChecker.cs b/src/Copybara.Core/Git/GerritApi/GerritApiTransportWithChecker.cs new file mode 100644 index 000000000..fcc5cb0d3 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/GerritApiTransportWithChecker.cs @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Checks; +using Copybara.Common; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Git.GerritApi; + +/// +/// A wrapper that runs an over each request +/// before delegating. Port of +/// com.google.copybara.git.gerritapi.GerritApiTransportWithChecker. +/// +/// +/// NOTE(port): the Java original delegates to a com.google.copybara.checks.ApiChecker. Since +/// that helper is not (yet) ported, this class inlines its convenience logic (build a field map from +/// the arguments and call ). +/// The response type, which upstream passes as a reflected Type, is represented here by the +/// generic argument and surfaced to the checker as a string. +/// +public class GerritApiTransportWithChecker : IGerritApiTransport +{ + private readonly IGerritApiTransport _delegate; + private readonly IChecker _checker; + private readonly Console _console; + + public GerritApiTransportWithChecker( + IGerritApiTransport @delegate, IChecker checker, Console console) + { + _delegate = Preconditions.CheckNotNull(@delegate); + _checker = Preconditions.CheckNotNull(checker); + _console = Preconditions.CheckNotNull(console); + } + + public Task GetAsync(string path) + { + Check( + ("path", path), + ("response_type", typeof(T).ToString())); + return _delegate.GetAsync(path); + } + + public Task PostAsync(string path, object request) + { + Check( + ("path", path), + ("request", request.ToString() ?? string.Empty), + ("response_type", typeof(T).ToString())); + return _delegate.PostAsync(path, request); + } + + public Task PutAsync(string path, object request) + { + Check( + ("path", path), + ("request", request.ToString() ?? string.Empty), + ("response_type", typeof(T).ToString())); + return _delegate.PutAsync(path, request); + } + + private void Check(params (string Field, string Value)[] fields) + { + var builder = ImmutableDictionary.CreateBuilder(); + foreach (var (field, value) in fields) + { + builder[field] = value; + } + + _checker.DoCheck(builder.ToImmutable(), _console); + } +} diff --git a/src/Copybara.Core/Git/GerritApi/GerritApiUtil.cs b/src/Copybara.Core/Git/GerritApi/GerritApiUtil.cs new file mode 100644 index 000000000..d27c6d987 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/GerritApiUtil.cs @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Globalization; + +namespace Copybara.Git.GerritApi; + +/// Utilities for dealing with Gerrit API. +public static class GerritApiUtil +{ + /// + /// Parses dates like "2014-12-21 17:30:08.000000000". + /// + /// + /// NOTE(port): Java uses DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.n") where + /// n is nano-of-second (up to 9 digits) and the zone is fixed to UTC. .NET's custom format + /// uses fffffffff for 9 fractional-second digits; parsing yields a UTC + /// . + /// + private const string TimestampFormat = "yyyy-MM-dd HH:mm:ss.fffffffff"; + + /// Parses a Gerrit timestamp into a UTC . + public static DateTimeOffset ParseTimestamp(string date) + { + // AssumeUniversal + AdjustToUniversal mirrors Java's .withZone(ZoneOffset.UTC). + var dt = DateTime.ParseExact( + date, + TimestampFormat, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + return new DateTimeOffset(dt, TimeSpan.Zero); + } +} diff --git a/src/Copybara.Core/Git/GerritApi/GerritEventType.cs b/src/Copybara.Core/Git/GerritApi/GerritEventType.cs new file mode 100644 index 000000000..98599d262 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/GerritEventType.cs @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Git.GerritApi; + +/// Type of events that we can monitor from Gerrit. +public enum GerritEventType +{ + /// Used if we don't know the event type. + UNKNOWN, + + LABELS, + SUBMIT_REQUIREMENTS, + STATUS, +} diff --git a/src/Copybara.Core/Git/GerritApi/GetChangeInput.cs b/src/Copybara.Core/Git/GerritApi/GetChangeInput.cs new file mode 100644 index 000000000..ae61ed803 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/GetChangeInput.cs @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; + +namespace Copybara.Git.GerritApi; + +/// +/// An object that represents the input parameters for get change: +/// +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#get-change +/// +public class GetChangeInput : IEquatable +{ + private readonly IReadOnlySet _include; + + public GetChangeInput() + : this(ImmutableHashSet.Empty) + { + } + + public GetChangeInput(IReadOnlySet include) + { + _include = include; + } + + public string AsUrlParams() => string.Join("&", _include.Select(i => "o=" + i)); + + public bool Equals(GetChangeInput? other) + { + if (ReferenceEquals(this, other)) + { + return true; + } + + if (other is null) + { + return false; + } + + return _include.SetEquals(other._include); + } + + public override bool Equals(object? o) => o is GetChangeInput other && Equals(other); + + public override int GetHashCode() + { + var hash = default(HashCode); + foreach (var i in _include.OrderBy(x => x)) + { + hash.Add(i); + } + + return hash.ToHashCode(); + } +} diff --git a/src/Copybara.Core/Git/GerritApi/GitPersonInfo.cs b/src/Copybara.Core/Git/GerritApi/GitPersonInfo.cs new file mode 100644 index 000000000..bd11fedcc --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/GitPersonInfo.cs @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#git-person-info +[StarlarkBuiltin("gerritapi.GitPersonInfo", Doc = "Git person information.")] +public class GitPersonInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("email")] + public string? Email { get; set; } + + [JsonPropertyName("date")] + public string? Date { get; set; } + + [JsonPropertyName("tz")] + public int Tz { get; set; } + + [StarlarkMethod( + "name", + Doc = "The name of the author/committer.", + StructField = true, + AllowReturnNones = true)] + public string? GetName() => Name; + + [StarlarkMethod( + "email", + Doc = "The email address of the author/committer.", + StructField = true, + AllowReturnNones = true)] + public string? GetEmail() => Email; + + public DateTimeOffset GetDate() => + GerritApiUtil.ParseTimestamp(Date!).ToOffset(TimeSpan.FromMinutes(Tz)); + + [StarlarkMethod( + "date", + Doc = "The timestamp of when this identity was constructed.", + StructField = true, + AllowReturnNones = true)] + public string? GetDateForSkylark() => Date; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"GitPersonInfo{{name={Name}, email={Email}, date={Date}, tz={Tz}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/IGerritApiTransport.cs b/src/Copybara.Core/Git/GerritApi/IGerritApiTransport.cs new file mode 100644 index 000000000..f56cdfd48 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/IGerritApiTransport.cs @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Exceptions; + +namespace Copybara.Git.GerritApi; + +/// +/// Http transport interface for talking to a Gerrit host. Port of +/// com.google.copybara.git.gerritapi.GerritApiTransport. +/// +/// +/// NOTE(port): Java passes a java.lang.reflect.Type responseType to each method. In the .NET +/// port the response type is expressed as a generic type argument T, which is both more +/// idiomatic and works directly with . Calls are async because the +/// underlying HTTP work is I/O bound. +/// +public interface IGerritApiTransport +{ + /// Do a http GET call. + /// + /// + Task GetAsync(string path); + + /// Do a http POST call. + /// + /// + Task PostAsync(string path, object request); + + /// Do a http PUT call. + /// + /// + Task PutAsync(string path, object request); +} diff --git a/src/Copybara.Core/Git/GerritApi/IncludeResult.cs b/src/Copybara.Core/Git/GerritApi/IncludeResult.cs new file mode 100644 index 000000000..932ee7d06 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/IncludeResult.cs @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Git.GerritApi; + +/// Fields to include in the response . +public enum IncludeResult +{ + LABELS, + DETAILED_LABELS, + CURRENT_REVISION, + ALL_REVISIONS, + DOWNLOAD_COMMANDS, + CURRENT_COMMIT, + ALL_COMMITS, + CURRENT_FILES, + ALL_FILES, + DETAILED_ACCOUNTS, + REVIEWER_UPDATES, + MESSAGES, + CURRENT_ACTIONS, + CHANGE_ACTIONS, + REVIEWED, + SUBMITTABLE, + WEB_LINKS, + CHECK, + COMMIT_FOOTERS, + PUSH_CERTIFICATES, + SUBMIT_REQUIREMENTS, +} diff --git a/src/Copybara.Core/Git/GerritApi/LabelInfo.cs b/src/Copybara.Core/Git/GerritApi/LabelInfo.cs new file mode 100644 index 000000000..cd7f476ed --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/LabelInfo.cs @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#label-info +[StarlarkBuiltin("gerritapi.LabelInfo", Doc = "Gerrit label information.")] +public class LabelInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("optional")] + public bool Optional { get; set; } + + [JsonPropertyName("approved")] + public AccountInfo? Approved { get; set; } + + [JsonPropertyName("rejected")] + public AccountInfo? Rejected { get; set; } + + [JsonPropertyName("recommended")] + public AccountInfo? Recommended { get; set; } + + [JsonPropertyName("disliked")] + public AccountInfo? Disliked { get; set; } + + [JsonPropertyName("blocking")] + public bool Blocking { get; set; } + + [JsonPropertyName("value")] + public int Value { get; set; } + + [JsonPropertyName("default_value")] + public int DefaultValue { get; set; } + + [JsonPropertyName("values")] + public IReadOnlyDictionary? Values { get; set; } + + [JsonPropertyName("all")] + public IReadOnlyList? All { get; set; } + + public bool IsOptional() => Optional; + + [StarlarkMethod( + "approved", + Doc = + "One user who approved this label on the change (voted the maximum value) as an " + + "AccountInfo entity.", + StructField = true, + AllowReturnNones = true)] + public AccountInfo? GetApproved() => Approved; + + [StarlarkMethod( + "rejected", + Doc = + "One user who rejected this label on the change (voted the minimum value) as an " + + "AccountInfo entity.", + StructField = true, + AllowReturnNones = true)] + public AccountInfo? GetRejected() => Rejected; + + [StarlarkMethod( + "recommended", + Doc = + "One user who recommended this label on the change (voted positively, but not the " + + "maximum value) as an AccountInfo entity.", + StructField = true, + AllowReturnNones = true)] + public AccountInfo? GetRecommended() => Recommended; + + [StarlarkMethod( + "disliked", + Doc = + "One user who disliked this label on the change (voted negatively, but not the " + + "minimum value) as an AccountInfo entity.", + StructField = true, + AllowReturnNones = true)] + public AccountInfo? GetDisliked() => Disliked; + + [StarlarkMethod( + "blocking", + Doc = "If true, the label blocks submit operation. If not set, the default is false.", + StructField = true)] + public bool IsBlocking() => Blocking; + + [StarlarkMethod( + "value", + Doc = + "The voting value of the user who recommended/disliked this label on the change if " + + "it is not `\"+1\"`/`\"-1\"`.", + StructField = true)] + public int GetValue() => Value; + + [StarlarkMethod( + "default_value", + Doc = + "The default voting value for the label. This value may be outside the range " + + "specified in permitted_labels.", + StructField = true)] + public int GetDefaultValue() => DefaultValue; + + [StarlarkMethod( + "values", + Doc = + "A map of all values that are allowed for this label. The map maps the values " + + "(`\"-2\"`, `\"-1\"`, `\"0\"`, `\"+1\"`, `\"+2\"`) to the value descriptions.", + StructField = true)] + public IReadOnlyDictionary GetValues() => + Values is null ? ImmutableDictionary.Empty : Values.ToImmutableDictionary(); + + [StarlarkMethod( + "all", + Doc = + "List of all approvals for this label as a list of ApprovalInfo entities. Items " + + "in this list may not represent actual votes cast by users; if a user votes on " + + "any label, a corresponding ApprovalInfo will appear in this list for all labels.", + StructField = true)] + public IReadOnlyList GetAll() => + All is not null ? All.ToImmutableArray() : ImmutableArray.Empty; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"LabelInfo{{optional={Optional}, approved={Approved}, rejected={Rejected}, " + + $"recommended={Recommended}, disliked={Disliked}, blocking={Blocking}, value={Value}, " + + $"defaultValue={DefaultValue}, values={Values}, all={All}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/NotifyType.cs b/src/Copybara.Core/Git/GerritApi/NotifyType.cs new file mode 100644 index 000000000..8d155c7a2 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/NotifyType.cs @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Git.GerritApi; + +/// Type of notification to send when abandoning/deleting a review or reviewer. +/// +/// NOTE(port): In the Java original, ALL is annotated with @NullValue, meaning it is +/// serialized as null (i.e. omitted). Callers mirror upstream by converting the enum to its +/// wire representation via before assigning it to a string field. +/// +public enum NotifyType +{ + NONE, + OWNER, + OWNER_REVIEWERS, + ALL, +} + +/// Helpers for translating to/from Gerrit wire values. +public static class NotifyTypeExtensions +{ + /// + /// Returns the wire representation of the notify type, or null for + /// which upstream annotates with @NullValue (i.e. omitted from the request). + /// + public static string? ToWireValue(this NotifyType notify) => + notify == NotifyType.ALL ? null : notify.ToString(); +} diff --git a/src/Copybara.Core/Git/GerritApi/ParentCommitInfo.cs b/src/Copybara.Core/Git/GerritApi/ParentCommitInfo.cs new file mode 100644 index 000000000..67883748c --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ParentCommitInfo.cs @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// Restricted version of for describing parents. +[StarlarkBuiltin("gerritapi.ParentCommitInfo", Doc = "Gerrit parent commit information.")] +public class ParentCommitInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("commit")] + public string? Commit { get; set; } + + [JsonPropertyName("subject")] + public string? Subject { get; set; } + + [StarlarkMethod( + "commit", + Doc = + "The commit ID. Not set if included in a RevisionInfo entity that is contained " + + "in a map which has the commit ID as key.", + StructField = true, + AllowReturnNones = true)] + public string? GetCommit() => Commit; + + [StarlarkMethod( + "subject", + Doc = "The subject of the commit (header line of the commit message).", + StructField = true, + AllowReturnNones = true)] + public string? GetSubject() => Subject; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"ParentCommitInfo{{commit={Commit}, subject={Subject}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/ProjectAccessInfo.cs b/src/Copybara.Core/Git/GerritApi/ProjectAccessInfo.cs new file mode 100644 index 000000000..dadcd2ceb --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ProjectAccessInfo.cs @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// +/// Relevant field(s) of the ProjectAccessInfo message. +/// https://gerrit-review.googlesource.com/Documentation/rest-api-access.html#project-access-info +/// +public class ProjectAccessInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("is_owner")] + public bool IsOwner { get; set; } + + public bool GetIsOwner() => IsOwner; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => $"ProjectAccessInfo{{is_owner={IsOwner}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/ProjectInfo.cs b/src/Copybara.Core/Git/GerritApi/ProjectInfo.cs new file mode 100644 index 000000000..2e4ffde08 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ProjectInfo.cs @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#project-info +public class ProjectInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("parent")] + public string? Parent { get; set; } + + [JsonPropertyName("description")] + public string? Description { get; set; } + + [JsonPropertyName("state")] + public string? StateString { get; set; } + + [JsonPropertyName("branches")] + public IReadOnlyDictionary? Branches { get; set; } + + public enum State + { + ACTIVE, + READ_ONLY, + HIDDEN, + } + + public string? GetId() => Id; + + public string? GetName() => Name; + + public string? GetParent() => Parent; + + public string? GetDescription() => Description; + + public State? GetState() => StateString is null ? null : Enum.Parse(StateString); + + public IReadOnlyDictionary? GetBranches() => Branches; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"ProjectInfo{{id={Id}, name={Name}, parent={Parent}, description={Description}, " + + $"state={StateString}, branches={Branches}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/ProjectInput.cs b/src/Copybara.Core/Git/GerritApi/ProjectInput.cs new file mode 100644 index 000000000..512889197 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ProjectInput.cs @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Git.GerritApi; + +/// +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#project-input +/// +public class ProjectInput +{ +} diff --git a/src/Copybara.Core/Git/GerritApi/RestoreInput.cs b/src/Copybara.Core/Git/GerritApi/RestoreInput.cs new file mode 100644 index 000000000..100cd9248 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/RestoreInput.cs @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace Copybara.Git.GerritApi; + +/// +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#restore-input +/// +public class RestoreInput +{ + [JsonPropertyName("message")] + public string? Message { get; set; } + + private RestoreInput(string? message) + { + Message = message; + } + + public static RestoreInput Create(string? message) => new(message); + + public static RestoreInput CreateWithoutComment() => new(null); +} diff --git a/src/Copybara.Core/Git/GerritApi/ReviewResult.cs b/src/Copybara.Core/Git/GerritApi/ReviewResult.cs new file mode 100644 index 000000000..5d5fbdc52 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ReviewResult.cs @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-result +[StarlarkBuiltin("gerritapi.ReviewResult", Doc = "Gerrit review result.")] +public class ReviewResult : IStarlarkPrintableValue +{ + [JsonPropertyName("labels")] + public IReadOnlyDictionary? Labels { get; set; } + + [JsonPropertyName("ready")] + public bool Ready { get; set; } + + public ReviewResult(IReadOnlyDictionary? labels, bool ready) + { + Labels = labels; + Ready = ready; + } + + public ReviewResult() + { + } + + [StarlarkMethod( + "labels", + Doc = "Map of labels to values after the review was posted.", + StructField = true)] + public IReadOnlyDictionary GetLabelsForStarlark() + { + var m = ImmutableDictionary.CreateBuilder(); + foreach (var e in GetLabels()) + { + m[e.Key] = StarlarkInt.Of(e.Value); + } + + return m.ToImmutable(); + } + + public IReadOnlyDictionary GetLabels() => + Labels is null ? ImmutableDictionary.Empty : Labels.ToImmutableDictionary(); + + [StarlarkMethod( + "ready", + Doc = + "If true, the change was moved from WIP to ready for review as a result of this action." + + " Not set if false.", + StructField = true)] + public bool IsReady() => Ready; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => $"ReviewResult{{labels={Labels}, ready={Ready}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/ReviewerInput.cs b/src/Copybara.Core/Git/GerritApi/ReviewerInput.cs new file mode 100644 index 000000000..8d958d302 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/ReviewerInput.cs @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Copybara.Common; + +namespace Copybara.Git.GerritApi; + +/// +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#reviewer-input +/// request json. +/// +public class ReviewerInput +{ + [JsonPropertyName("reviewer")] + public string Reviewer { get; set; } + + public ReviewerInput(string reviewer) + { + Reviewer = Preconditions.CheckNotNull(reviewer); + } +} diff --git a/src/Copybara.Core/Git/GerritApi/RevisionInfo.cs b/src/Copybara.Core/Git/GerritApi/RevisionInfo.cs new file mode 100644 index 000000000..97fe7efda --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/RevisionInfo.cs @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#revision-info +[StarlarkBuiltin("gerritapi.RevisionInfo", Doc = "Gerrit revision information.")] +public class RevisionInfo : IStarlarkPrintableValue +{ + [JsonPropertyName("kind")] + public string? KindString { get; set; } + + [JsonPropertyName("_number")] + public int PatchsetNumber { get; set; } + + [JsonPropertyName("created")] + public string? Created { get; set; } + + [JsonPropertyName("uploader")] + public AccountInfo? Uploader { get; set; } + + [JsonPropertyName("ref")] + public string? Ref { get; set; } + + [JsonPropertyName("fetch")] + public IReadOnlyDictionary? Fetch { get; set; } + + [JsonPropertyName("commit")] + public CommitInfo? Commit { get; set; } + + public RevisionKind GetKind() => Enum.Parse(KindString!); + + [StarlarkMethod( + "kind", + Doc = + "The change kind. Valid values are REWORK, TRIVIAL_REBASE, MERGE_FIRST_PARENT_UPDATE, " + + "NO_CODE_CHANGE, and NO_CHANGE.", + StructField = true, + AllowReturnNones = true)] + public string? GetKindAsString() => KindString; + + [StarlarkMethod( + "patchset_number", + Doc = "The patch set number, or edit if the patch set is an edit.", + StructField = true)] + public int GetPatchsetNumber() => PatchsetNumber; + + [StarlarkMethod( + "created", + Doc = "The timestamp of when the patch set was created.", + StructField = true, + AllowReturnNones = true)] + public string? GetCreated() => Created; + + [StarlarkMethod( + "uploader", + Doc = "The uploader of the patch set as an AccountInfo entity.", + StructField = true, + AllowReturnNones = true)] + public AccountInfo? GetUploader() => Uploader; + + [StarlarkMethod( + "ref", + Doc = "The Git reference for the patch set.", + StructField = true, + AllowReturnNones = true)] + public string? GetRef() => Ref; + + public IReadOnlyDictionary GetFetch() => + Fetch is null ? ImmutableDictionary.Empty : Fetch.ToImmutableDictionary(); + + [StarlarkMethod( + "commit", + Doc = "The commit of the patch set as CommitInfo entity.", + StructField = true, + AllowReturnNones = true)] + public CommitInfo? GetCommit() => Commit; + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); + + public override string ToString() => + $"RevisionInfo{{kind={KindString}, patchsetNumber={PatchsetNumber}, created={Created}, " + + $"uploader={Uploader}, ref={Ref}, fetch={Fetch}, commit={Commit}}}"; +} + +/// +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#revision-info +/// +/// NOTE(port): Java nests this as RevisionInfo.Kind; here it is a sibling type named +/// RevisionKind to avoid a naming collision within the same namespace. +public enum RevisionKind +{ + REWORK, + TRIVIAL_REBASE, + MERGE_FIRST_PARENT_UPDATE, + NO_CODE_CHANGE, + NO_CHANGE, +} diff --git a/src/Copybara.Core/Git/GerritApi/SetReviewInput.cs b/src/Copybara.Core/Git/GerritApi/SetReviewInput.cs new file mode 100644 index 000000000..9c715dab5 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/SetReviewInput.cs @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input. +/// +[StarlarkBuiltin( + "SetReviewInput", + Doc = + "Input for posting a review to Gerrit. See " + + "https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input")] +public class SetReviewInput : IStarlarkPrintableValue, IEquatable +{ + [JsonPropertyName("message")] + public string? Message { get; set; } + + [JsonPropertyName("labels")] + public IReadOnlyDictionary Labels { get; set; } + + [JsonPropertyName("tag")] + public string? Tag { get; set; } + + /// The notify type, serialized to its Gerrit wire value (null for ALL). + [JsonPropertyName("notify")] + public string? NotifyWire => NotifyType?.ToWireValue(); + + [JsonIgnore] + public NotifyType? NotifyType { get; set; } + + public SetReviewInput() + { + Labels = new Dictionary(); + } + + private SetReviewInput( + string? message, IReadOnlyDictionary labels, string? tag, NotifyType notify) + { + Message = message; + Labels = labels; + Tag = tag; + NotifyType = notify; + } + + public SetReviewInput(string? message, IReadOnlyDictionary labels) + : this(message, labels, null, global::Copybara.Git.GerritApi.NotifyType.ALL) + { + } + + public static SetReviewInput Create( + string? message, IReadOnlyDictionary labels, string? tag) => + Create(message, labels, tag, global::Copybara.Git.GerritApi.NotifyType.ALL); + + public static SetReviewInput Create( + string? message, IReadOnlyDictionary labels, string? tag, NotifyType notify) => + new(message, labels, tag, notify); + + public string? GetMessage() => Message; + + public IReadOnlyDictionary GetLabels() => Labels; + + public NotifyType? GetNotify() => NotifyType; + + public string? GetTag() => Tag; + + public override string ToString() => + $"SetReviewInput{{message={Message}, labels={Labels}, tag={Tag}}}"; + + public bool Equals(SetReviewInput? other) + { + if (ReferenceEquals(this, other)) + { + return true; + } + + if (other is null) + { + return false; + } + + return Message == other.Message + && LabelsEqual(Labels, other.Labels) + && Tag == other.Tag; + } + + public override bool Equals(object? o) => o is SetReviewInput other && Equals(other); + + public override int GetHashCode() + { + var hash = default(HashCode); + hash.Add(Message); + foreach (var e in Labels) + { + hash.Add(e.Key); + hash.Add(e.Value); + } + + return hash.ToHashCode(); + } + + private static bool LabelsEqual(IReadOnlyDictionary a, IReadOnlyDictionary b) + { + if (a.Count != b.Count) + { + return false; + } + + foreach (var e in a) + { + if (!b.TryGetValue(e.Key, out var v) || v != e.Value) + { + return false; + } + } + + return true; + } + + public void Repr(Printer printer, StarlarkSemantics semantics) => printer.Append(ToString()); +} diff --git a/src/Copybara.Core/Git/GerritApi/SubmitInput.cs b/src/Copybara.Core/Git/GerritApi/SubmitInput.cs new file mode 100644 index 000000000..3a00b0c9e --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/SubmitInput.cs @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace Copybara.Git.GerritApi; + +/// +/// See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#submit-change +/// request json. +/// NotifyInfo (notify_details) not included for now. +/// on_behalf_of not included for now until we have a use case. +/// +public class SubmitInput +{ + [JsonPropertyName("notify")] + public string? Notify { get; set; } + + public SubmitInput(NotifyType? notify) + { + Notify = notify?.ToWireValue(); + } +} diff --git a/src/Copybara.Core/Git/GerritApi/SubmitRequirementExpressionInfo.cs b/src/Copybara.Core/Git/GerritApi/SubmitRequirementExpressionInfo.cs new file mode 100644 index 000000000..568a0bc41 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/SubmitRequirementExpressionInfo.cs @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// +/// Result of evaluating a single submit requirement expression. This API entity is populated from +/// Gerrit's SubmitRequirementExpressionResult. +/// +[StarlarkBuiltin( + "gerritapi.SubmitRequirementExpressionInfo", + Doc = "Result of evaluating submit requirement expression")] +public class SubmitRequirementExpressionInfo : IStarlarkValue +{ + [JsonPropertyName("expression")] + public string? Expression { get; set; } + + [JsonPropertyName("status")] + public string? StatusString { get; set; } + + [JsonPropertyName("fulfilled")] + public bool Fulfilled { get; set; } + + [StarlarkMethod( + "expression", + Doc = "The submit requirement expression as a string.", + StructField = true, + AllowReturnNones = true)] + public string? GetExpression() => Expression; + + public SubmitRequirementExpressionStatus GetStatus() => + Enum.Parse(StatusString!); + + [StarlarkMethod( + "status", + Doc = "The status of the submit requirement evaluation.", + StructField = true, + AllowReturnNones = true)] + public string? GetStatusAsString() => StatusString; + + [StarlarkMethod( + "fulfilled", + Doc = + "If true, this submit requirement result was created from a legacy SubmitRecord." + + " Otherwise, it was created by evaluating a submit requirement.", + StructField = true)] + public bool GetFulfilled() => Fulfilled; + + public override string ToString() => + $"SubmitRequirementExpressionInfo{{expression={Expression}, status={StatusString}, " + + $"fulfilled={Fulfilled}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/SubmitRequirementExpressionStatus.cs b/src/Copybara.Core/Git/GerritApi/SubmitRequirementExpressionStatus.cs new file mode 100644 index 000000000..8e7893d32 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/SubmitRequirementExpressionStatus.cs @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Git.GerritApi; + +/// Status describing the result of evaluating the submit requirement expression. +public enum SubmitRequirementExpressionStatus +{ + PASS, + FAIL, + ERROR, + NOT_EVALUATED, +} diff --git a/src/Copybara.Core/Git/GerritApi/SubmitRequirementInput.cs b/src/Copybara.Core/Git/GerritApi/SubmitRequirementInput.cs new file mode 100644 index 000000000..2aa281494 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/SubmitRequirementInput.cs @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// +/// An object that represents the input parameters for a submit requirement: +/// +/// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#submit-requirement-input +/// +public class SubmitRequirementInput : IStarlarkValue +{ + /// Submit requirement name. + [JsonPropertyName("name")] + public string Name { get; set; } + + /// + /// Query expression that can be evaluated on any change. If evaluated to true on a change, the + /// submit requirement is fulfilled and not blocking change submission. + /// + [JsonPropertyName("submittability_expression")] + public string SubmittabilityExpression { get; set; } + + public SubmitRequirementInput(string name, string submittabilityExpression) + { + Name = name; + SubmittabilityExpression = submittabilityExpression; + } + + [StarlarkMethod("name", Doc = "The submit requirement name.", StructField = true)] + public string GetName() => Name; + + [StarlarkMethod( + "submittability_expression", + Doc = + "Query expression that can be evaluated on any change. If evaluated to true on a change," + + " the submit requirement is fulfilled and not blocking change submission.", + StructField = true)] + public string GetSubmittabilityExpression() => SubmittabilityExpression; +} diff --git a/src/Copybara.Core/Git/GerritApi/SubmitRequirementResultInfo.cs b/src/Copybara.Core/Git/GerritApi/SubmitRequirementResultInfo.cs new file mode 100644 index 000000000..76e4cd6a5 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/SubmitRequirementResultInfo.cs @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GerritApi; + +/// Result of evaluating a submit requirement on a change. +public class SubmitRequirementResultInfo : IStarlarkValue +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("status")] + public string? StatusString { get; set; } + + [JsonPropertyName("submittability_expression_result")] + public SubmitRequirementExpressionInfo? SubmittabilityExpressionResult { get; set; } + + [JsonPropertyName("is_legacy")] + public bool IsLegacy { get; set; } + + [StarlarkMethod("name", Doc = "The submit requirement name.", StructField = true)] + public string? GetName() => Name; + + public SubmitRequirementResultStatus GetStatus() => + Enum.Parse(StatusString!); + + [StarlarkMethod( + "status", + Doc = "The status of the submit requirement evaluation.", + StructField = true)] + public string? GetStatusAsString() => StatusString; + + [StarlarkMethod( + "is_legacy", + Doc = + "If true, this submit requirement result was created from a legacy SubmitRecord." + + " Otherwise, it was created by evaluating a submit requirement.", + StructField = true)] + public bool GetIsLegacy() => IsLegacy; + + [StarlarkMethod( + "submittability_expression_result", + Doc = + "A SubmitRequirementExpressionInfo containing the result of evaluating the" + + " submittabilityexpression. If the submit requirement does not apply, the status" + + " field of the result will be set to NOT_EVALUATED.", + StructField = true)] + public SubmitRequirementExpressionInfo? GetSubmittabilityExpressionResult() => + SubmittabilityExpressionResult; + + public override string ToString() => + $"SubmitRequirementResultInfo{{name={Name}, status={StatusString}, " + + $"submittabilityExpressionResult={SubmittabilityExpressionResult}, isLegacy={IsLegacy}}}"; +} diff --git a/src/Copybara.Core/Git/GerritApi/SubmitRequirementResultStatus.cs b/src/Copybara.Core/Git/GerritApi/SubmitRequirementResultStatus.cs new file mode 100644 index 000000000..71598eec4 --- /dev/null +++ b/src/Copybara.Core/Git/GerritApi/SubmitRequirementResultStatus.cs @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Git.GerritApi; + +/// Status describing the result of evaluating the submit requirement. +public enum SubmitRequirementResultStatus +{ + SATISFIED, + UNSATISFIED, + OVERRIDDEN, + NOT_APPLICABLE, + ERROR, + FORCED, +} diff --git a/src/Copybara.Core/Git/GerritChange.cs b/src/Copybara.Core/Git/GerritChange.cs new file mode 100644 index 000000000..d6e3eb42d --- /dev/null +++ b/src/Copybara.Core/Git/GerritChange.cs @@ -0,0 +1,295 @@ +/* + * Copyright (C) 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.RegularExpressions; +using Copybara.Common; +using Copybara.Exceptions; + +namespace Copybara.Git; + +/// +/// A class that represents a Gerrit change. It contains all the necessary objects to do a fetch when +/// is invoked. Port of com.google.copybara.git.GerritChange. +/// +internal sealed class GerritChange +{ + public const string GerritChangeNumberLabel = "GERRIT_CHANGE_NUMBER"; + public const string GerritChangeIdLabel = "GERRIT_CHANGE_ID"; + public const string GerritCompleteChangeIdLabel = "GERRIT_COMPLETE_CHANGE_ID"; + public const string GerritChangeUrlLabel = "GERRIT_CHANGE_URL"; + public const string GerritChangeBranch = "GERRIT_CHANGE_BRANCH"; + public const string GerritChangeTopic = "GERRIT_CHANGE_TOPIC"; + public const string GerritChangeDescriptionLabel = "GERRIT_CHANGE_DESCRIPTION"; + public const string GerritOwnerEmailLabel = "GERRIT_OWNER_EMAIL"; + public const string GerritOwnerUsernameLabel = "GERRIT_OWNER_USERNAME"; + private const string GerritPatchSetRefPrefix = "PatchSet "; + + // Mirrors GitModule.DEFAULT_INTEGRATE_LABEL (GitModule is ported separately). + internal const string DefaultIntegrateLabel = "COPYBARA_INTEGRATE_REVIEW"; + + private static readonly Regex WholeGerritRef = + new("^refs/changes/[0-9]{2}/([0-9]+)/([0-9]+)$", RegexOptions.Compiled); + + private static readonly Regex UrlPattern = + new(@"^https?://.*?/([0-9]+)(?:/([0-9]+))?/?$", RegexOptions.Compiled); + + private readonly GitRepository _repository; + private readonly GeneralOptions _generalOptions; + private readonly string _repoUrl; + private readonly int _change; + private readonly int _patchSet; + private readonly string _ref; + + private GerritChange( + GitRepository repository, + GeneralOptions generalOptions, + string repoUrl, + int change, + int patchSet, + string @ref) + { + _repository = Preconditions.CheckNotNull(repository); + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _repoUrl = repoUrl; + _change = change; + _patchSet = patchSet; + _ref = @ref; + } + + /// Get the change number. + public int GetChange() => _change; + + /// Gets the specific PatchSet of the Change. + public int GetPatchSet() => _patchSet; + + /// Context reference for creating GitRevision. + public string GetRef() => _ref; + + /// + /// Given a local repository, a repo url and a reference, it tries to do its best to resolve the + /// reference to a Gerrit Change. + /// + /// Note that if the PatchSet is not found in the ref, it will go to Gerrit to get the latest + /// PatchSet number. + /// + /// a Gerrit change if it can be resolved. Null otherwise. + public static GerritChange? Resolve( + GitRepository repository, string repoUrl, string @ref, GeneralOptions options) + { + if (string.IsNullOrEmpty(@ref)) + { + return null; + } + Match refMatcher = WholeGerritRef.Match(@ref); + if (refMatcher.Success) + { + return new GerritChange( + repository, + options, + repoUrl, + int.Parse(refMatcher.Groups[1].Value), + int.Parse(refMatcher.Groups[2].Value), + @ref); + } + // A change number like '23423' + if (@ref.Length > 0 && @ref.All(char.IsDigit)) + { + return ResolveLatestPatchSet(repository, options, repoUrl, int.Parse(@ref)); + } + + Match urlMatcher = UrlPattern.Match(@ref); + if (!urlMatcher.Success) + { + return null; + } + + if (!@ref.StartsWith(repoUrl, StringComparison.Ordinal)) + { + // Assume it is our url. We can make this more strict later + options.GetConsole().Warn( + $"Assuming repository '{repoUrl}' for looking for review '{@ref}'"); + } + int change = int.Parse(urlMatcher.Groups[1].Value); + int? patchSet = + urlMatcher.Groups[2].Success && urlMatcher.Groups[2].Value.Length > 0 + ? int.Parse(urlMatcher.Groups[2].Value) + : null; + if (patchSet == null) + { + return ResolveLatestPatchSet(repository, options, repoUrl, change); + } + var patchSets = GetGerritPatchSets(repository, repoUrl, change); + if (!patchSets.ContainsKey(patchSet.Value)) + { + throw new CannotResolveRevisionException( + $"Cannot find patch set {patchSet} for change {change} in {repoUrl}. Available Patch" + + $" sets: {string.Join(", ", patchSets.Keys)}"); + } + return new GerritChange( + repository, options, repoUrl, change, patchSet.Value, + patchSets[patchSet.Value].ContextReference()!); + } + + /// Fetch the change from Gerrit. + /// additional labels to add to the GitRevision labels. + /// The resolved and fetched SHA-1 of the change. + public GitRevision Fetch(ImmutableListMultimap additionalLabels) + { + string metaRef = $"refs/changes/{_change % 100:D2}/{_change}/meta"; + _repository.Fetch( + _repoUrl, + prune: true, + force: true, + new[] { _ref + ":refs/gerrit/" + _ref, metaRef + ":refs/gerrit/" + metaRef }, + partialFetch: false, + depth: null, + tags: false); + GitRevision gitRevision = _repository.ResolveReference("refs/gerrit/" + _ref); + GitRevision metaRevision = _repository.ResolveReference("refs/gerrit/" + metaRef); + string changeId = GetChangeIdFromMeta(_repository, metaRevision, metaRef); + string changeNumber = _change.ToString(); + string changeDescription = GetDescriptionFromMeta(_repository, metaRevision, metaRef); + + var labels = ImmutableListMultimap.CreateBuilder(); + labels.Put(GerritChangeNumberLabel, changeNumber); + labels.Put(GerritChangeIdLabel, changeId); + labels.Put(GerritChangeDescriptionLabel, changeDescription); + labels.Put( + DefaultIntegrateLabel, + new GerritIntegrateLabel( + _repository, _generalOptions, _repoUrl, _change, _patchSet, changeId).ToString()); + labels.PutAll(additionalLabels); + foreach (var e in _generalOptions.CliLabels()) + { + labels.Put(e.Key, e.Value); + } + + return new GitRevision( + _repository, + gitRevision.GetHash(), + GerritPatchSetAsReviewReference(_patchSet), + changeNumber, + labels.Build(), + _repoUrl); + } + + private static GerritChange ResolveLatestPatchSet( + GitRepository repository, GeneralOptions options, string repoUrl, int changeNumber) + { + // Last entry is the latest patchset, since it is ordered by patchsetId. + var patchSets = GetGerritPatchSets(repository, repoUrl, changeNumber); + var lastPatchset = patchSets[patchSets.Keys.Last()]; + int lastKey = patchSets.Keys.Last(); + return new GerritChange( + repository, options, repoUrl, changeNumber, lastKey, lastPatchset.ContextReference()!); + } + + /// + /// Use NoteDB for extracting the Change-id. It should be the first commit in the log of the meta + /// reference. + /// + private static string GetChangeIdFromMeta( + GitRepository repo, GitRevision metaRevision, string metaRef) + { + var changes = GetChanges(repo, metaRevision, metaRef); + string? changeId = null; + foreach (var change in changes[^1].GetLabels()) + { + if (change.IsLabel() && change.GetName().Equals("Change-id") + && change.GetSeparator().Equals(": ")) + { + changeId = change.GetValue(); + } + } + if (changeId == null) + { + throw new RepoException( + $"Cannot find Change-id in {metaRef}. Not present in: \n{changes[^1].GetText()}"); + } + + return changeId; + } + + private static string GetDescriptionFromMeta( + GitRepository repo, GitRevision metaRevision, string metaRef) => + GetChanges(repo, metaRevision, metaRef)[0].GetText(); + + /// + /// Returns the list of s. Guarantees that there is at least one change. + /// + private static IReadOnlyList GetChanges( + GitRepository repo, GitRevision metaRevision, string metaRef) + { + var changes = repo.Log(metaRevision.GetHash()).Run() + .Select(e => ChangeMessage.ParseMessage(e.Body ?? "")) + .ToList(); + + if (changes.Count == 0) + { + throw new RepoException("Cannot find any PatchSet in " + metaRef); + } + return changes; + } + + /// + /// Get all the patchsets for a change ordered by the patchset number. Last is the most recent one. + /// + public static SortedDictionary GetGerritPatchSets( + GitRepository repository, string url, int changeNumber) + { + var patchSets = new SortedDictionary(); + string basePath = $"refs/changes/{changeNumber % 100:D2}/{changeNumber}"; + var refsToSha1 = repository.LsRemote(url, new[] { basePath + "/*" }); + if (refsToSha1.Count == 0) + { + throw new CannotResolveRevisionException( + $"Cannot find change number {changeNumber} in '{url}'"); + } + foreach (var e in refsToSha1) + { + if (e.Key.EndsWith("/meta", StringComparison.Ordinal) + || e.Key.EndsWith("/robot-comments", StringComparison.Ordinal)) + { + continue; + } + Preconditions.CheckState( + e.Key.StartsWith(basePath + "/", StringComparison.Ordinal), + "Unexpected response reference {0} for {1}", + e.Key, + basePath); + Match matcher = WholeGerritRef.Match(e.Key); + Preconditions.CheckArgument( + matcher.Success, + "Unexpected format for response reference {0} for {1}", + e.Key, + basePath); + int patchSet = int.Parse(matcher.Groups[2].Value); + patchSets[patchSet] = + new GitRevision( + repository, + e.Value, + GerritPatchSetAsReviewReference(patchSet), + e.Key, + ImmutableListMultimap.Empty, + url); + } + return patchSets; + } + + internal static string GerritPatchSetAsReviewReference(int patchSet) => + GerritPatchSetRefPrefix + patchSet; +} diff --git a/src/Copybara.Core/Git/GerritDestination.cs b/src/Copybara.Core/Git/GerritDestination.cs new file mode 100644 index 000000000..f599a5a6a --- /dev/null +++ b/src/Copybara.Core/Git/GerritDestination.cs @@ -0,0 +1,700 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using Copybara.Authoring; +using Copybara.Checks; +using Copybara.Common; +using Copybara.Effect; +using Copybara.Exceptions; +using Copybara.Git.GerritApi; +using Copybara.Revision; +using Copybara.Util; +using Console = Copybara.Util.Console.Console; +using GerritApiClient = Copybara.Git.GerritApi.GerritApi; + +namespace Copybara.Git; + +/// +/// Gerrit repository destination. Port of com.google.copybara.git.GerritDestination. +/// +public sealed class GerritDestination : IDestination +{ + internal const int MaxFindAttempts = 150; + private const int SubmitMaxRetryDelayMs = 30000; + + internal const string ChangeIdLabel = "Change-Id"; + + private readonly GitDestination _gitDestination; + private readonly bool _submit; + + private GerritDestination(GitDestination gitDestination, bool submit) + { + _gitDestination = Preconditions.CheckNotNull(gitDestination); + _submit = submit; + } + + public override string ToString() => $"GerritDestination{{gitDestination={_gitDestination}}}"; + + /// What to do in the presence or absent of Change-Id in message. + public enum ChangeIdPolicy + { + /// Require that the change_id is present in the message as a valid label. + Require, + + /// Fail if found in message. + FailIfPresent, + + /// Reuse if present. Otherwise generate a new one. + Reuse, + + /// Replace with a new one if found. + Replace, + } + + /// + /// Notify Gerrit push option: + /// https://gerrit-review.googlesource.com/Documentation/user-upload.html#notify + /// + public enum NotifyOption + { + None, + Owner, + OwnerReviewers, + All, + } + + /// A message info that contains also information if the change is a new review. + internal sealed class GerritMessageInfo : GitDestination.MessageInfo + { + internal readonly bool NewReview; + internal readonly string ChangeId; + + internal GerritMessageInfo( + IReadOnlyList labelsToAdd, bool newReview, string changeId) + : base(labelsToAdd) + { + NewReview = newReview; + ChangeId = Preconditions.CheckNotNull(changeId); + } + } + + internal sealed class GerritWriteHook : GitDestination.IWriteHook + { + private static readonly Regex GerritUrlLine = + new(@".*: *(http(s)?://[^ ]+)( .*)?", RegexOptions.Compiled); + + private static readonly Regex UserErrorRegexPattern = + new(@"(2 is restricted)|(submit requirement[\w-,.:!' ]*is unsatisfied)", + RegexOptions.Compiled); + + private readonly GerritOptions _gerritOptions; + private readonly string _repoUrl; + private readonly Author _committer; + private readonly IReadOnlyList _reviewersTemplate; + private readonly IChecker? _endpointChecker; + private readonly NotifyOption? _notifyOption; + private readonly Console _console; + private readonly ChangeIdPolicy _changeIdPolicy; + private readonly bool _allowEmptyDiffPatchSet; + private readonly GeneralOptions _generalOptions; + private readonly IReadOnlyList _ccTemplate; + private readonly IReadOnlyList _labelsTemplate; + private readonly string? _topicTemplate; + private readonly bool _partialFetch; + private readonly bool _gerritSubmit; + private readonly bool _primaryBranchMigrationMode; + + internal GerritWriteHook( + GeneralOptions generalOptions, + GerritOptions gerritOptions, + string repoUrl, + Author committer, + IReadOnlyList reviewersTemplate, + IReadOnlyList ccTemplate, + ChangeIdPolicy changeIdPolicy, + bool allowEmptyDiffPatchSet, + IReadOnlyList labelsTemplate, + IChecker? endpointChecker, + NotifyOption? notifyOption, + string? topicTemplate, + bool partialFetch, + bool gerritSubmit, + bool primaryBranchMigrationMode) + { + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _gerritOptions = Preconditions.CheckNotNull(gerritOptions); + _repoUrl = Preconditions.CheckNotNull(repoUrl); + _committer = Preconditions.CheckNotNull(committer); + _console = Preconditions.CheckNotNull(generalOptions.GetConsole()); + _changeIdPolicy = changeIdPolicy; + _allowEmptyDiffPatchSet = allowEmptyDiffPatchSet; + _reviewersTemplate = Preconditions.CheckNotNull(reviewersTemplate); + _ccTemplate = Preconditions.CheckNotNull(ccTemplate); + _endpointChecker = endpointChecker; + _notifyOption = notifyOption; + _labelsTemplate = labelsTemplate; + _topicTemplate = topicTemplate; + _partialFetch = partialFetch; + _gerritSubmit = gerritSubmit; + _primaryBranchMigrationMode = primaryBranchMigrationMode; + } + + public GitDestination.MessageInfo GenerateMessageInfo(TransformResult result) + { + if (!string.IsNullOrEmpty(_gerritOptions.GerritChangeId)) + { + // CLI flag always wins. + return CreateMessageInfo( + result, newReview: false, _gerritOptions.GerritChangeId, ChangeIdPolicy.Replace); + } + + string hashTag = ComputeInternalHashTag(result); + ChangeInfo? activeChange = FindActiveChange(hashTag); + if (activeChange != null) + { + return CreateMessageInfo( + result, newReview: false, activeChange.GetChangeId()!, _changeIdPolicy); + } + + // If no change is found, create a random change-id. Change-Ids can be reused later by + // looking by hashtag in the code above. + return CreateMessageInfo( + result, + newReview: true, + "I" + Sha1Hex(Guid.NewGuid().ToString()), + _changeIdPolicy); + } + + private static string Sha1Hex(string input) + { + byte[] hash = SHA1.HashData(Encoding.UTF8.GetBytes(input)); + return Convert.ToHexStringLower(hash); + } + + private string ComputeInternalHashTag(TransformResult result) => + "copybara_id_" + result.GetChangeIdentity() + + "_" + Regex.Replace(_committer.Email, "[ ,]", "_"); + + private ChangeInfo? FindActiveChange(string hashTag) + { + _console.ProgressFmt( + "Querying Gerrit ('{0}') for active changes with hashtag '{1}'", _repoUrl, hashTag); + IReadOnlyList changes; + try + { + changes = _gerritOptions.NewGerritApi(_repoUrl).GetChangesAsync(new ChangesQuery( + $"hashtag:\"{hashTag}\" AND project:{_gerritOptions.GetProject(_repoUrl)} AND" + + " status:NEW")) + .GetAwaiter().GetResult(); + } + catch (Exception e) when (e is RepoException or ValidationException) + { + const string errMsgFmt = "Failed querying the hash tag from gerrit changes. Reason: {0}"; + if (_generalOptions.DryRunMode) + { + _console.WarnFmt(errMsgFmt, e.Message); + return null; + } + throw; + } + ChangeInfo? maxChangeNumber = changes + .OrderByDescending(c => c.GetNumber()) + .FirstOrDefault(); + if (changes.Count > 1) + { + _console.WarnFmt( + "Multiple changes found for the same internal copybara tag: {0}. Reusing {1}", + string.Join(", ", changes.Select(c => c.GetNumber())), + maxChangeNumber!.GetNumber()); + } + return maxChangeNumber; + } + + private IReadOnlyList FindChanges( + string changeId, IEnumerable includes) + { + _console.ProgressFmt("Querying Gerrit ('{0}') for change '{1}'", _repoUrl, changeId); + return _gerritOptions.NewGerritApi(_repoUrl).GetChangesAsync(new ChangesQuery( + "change: " + changeId + " AND project:" + _gerritOptions.GetProject(_repoUrl)) + .WithInclude(includes)) + .GetAwaiter().GetResult(); + } + + public void BeforePush( + GitRepository repo, + GitDestination.MessageInfo messageInfo, + bool skipPush, + IReadOnlyList integrateLabels, + IReadOnlyList originChanges) + { + var gerritMessageInfo = (GerritMessageInfo)messageInfo; + if (_generalOptions.AllowEmptyDiffValue(_allowEmptyDiffPatchSet) || gerritMessageInfo.NewReview) + { + return; + } + using (_generalOptions.Profiler().Start("previous_patchset_check")) + { + ChangeInfo? changeInfo = FindChange(gerritMessageInfo.ChangeId); + if (changeInfo == null) + { + return; + } + var sameGitTree = new SameGitTree(repo, _repoUrl, _generalOptions, _partialFetch); + if (changeInfo.GetCurrentRevision() != null + && sameGitTree.HasSameTree(changeInfo.GetCurrentRevision()!)) + { + throw new RedundantChangeException( + $"Skipping creating a new Gerrit PatchSet for change {_repoUrl}/q/" + + $"{changeInfo.GetNumber()} since the diff is the same from the previous" + + $" PatchSet ({changeInfo.GetCurrentRevision()})", + changeInfo.GetCurrentRevision()!); + } + } + } + + private ChangeInfo? FindChange(string changeId) => FindChange(changeId, 0); + + private ChangeInfo? FindChange(string changeId, int maxDelay) + { + int currentAttempt = 0; + long delayMs; + do + { + var changes = FindChanges( + changeId, + new[] { IncludeResult.CURRENT_REVISION, IncludeResult.SUBMITTABLE }); + if (changes.Count != 0) + { + return changes[0]; + } + + delayMs = 1000 * (long)Math.Pow(2, currentAttempt); + if (delayMs <= maxDelay) + { + currentAttempt++; + _console.WarnFmt( + "Gerrit change {0} not found (attempt {1}). Retrying in {2} ms...", + changeId, currentAttempt, delayMs); + _gerritOptions.GetSleeper().Sleep(delayMs); + } + } + while (delayMs <= maxDelay); + + return null; + } + + private void SubmitChange(string changeId) + { + using (_generalOptions.Profiler().Start("submit_gerrit_change")) + { + try + { + ChangeInfo? changeInfo = FindChange(changeId, SubmitMaxRetryDelayMs); + if (changeInfo == null) + { + _console.WarnFmt( + "Gerrit change {0} still not found after waiting {1} milliseconds." + + " Skipping submit.", + changeId, SubmitMaxRetryDelayMs); + return; + } + GerritApiClient gerritApi = _gerritOptions.NewGerritApi(_repoUrl); + // If the change isn't yet submittable, try voting Code-Review+2 to approve it. + if (!changeInfo.IsSubmittable()) + { + try + { + gerritApi.SetReviewAsync( + changeInfo.GetChangeId()!, + changeInfo.GetCurrentRevision()!, + new SetReviewInput( + "", new Dictionary { ["Code-Review"] = 2 })) + .GetAwaiter().GetResult(); + } + catch (Exception e) when (e is RepoException or ValidationException) + { + _console.WarnFmt( + "Failed voting Code-Review + 2 to make change submittable:\n{0}", + e.Message); + } + } + string tripletId = changeInfo.GetTripletId()!; + gerritApi.SubmitChangeAsync(tripletId, new SubmitInput(null)) + .GetAwaiter().GetResult(); + _console.InfoFmt("Submitted change : {0}/changes/{1}", _repoUrl, tripletId); + } + catch (RepoException e) + { + if (UserErrorRegexPattern.IsMatch(e.Message)) + { + throw new ValidationException(e.Message, e); + } + throw; + } + } + } + + public string GetPushReference( + GitRepository repo, string pushToRefsFor, TransformResult transformResult) + { + string[] components = pushToRefsFor.Split('%', 2); + + var options = new List>(); + if (components.Length > 1) + { + foreach (var entry in components[1].Split(',')) + { + string[] strings = entry.Split('=', 2); + options.Add(strings.Length > 1 + ? new KeyValuePair(strings[0], strings[1]) + : new KeyValuePair(entry, null)); + } + } + + if (_notifyOption != null) + { + options.Add(new KeyValuePair("notify", _notifyOption.ToString())); + } + + string? topic = null; + if (_topicTemplate != null) + { + topic = LabelFinder.MapLabels( + transformResult.GetLabelFinder(), _topicTemplate, "topic"); + } + if (!string.IsNullOrEmpty(_gerritOptions.GerritTopic)) + { + if (topic != null) + { + _console.WarnFmt("Overriding topic {0} with {1}", topic, _gerritOptions.GerritTopic); + } + topic = _gerritOptions.GerritTopic; + } + + if (topic != null) + { + options.Add(new KeyValuePair("topic", topic)); + } + + if (pushToRefsFor.StartsWith("refs/for/", StringComparison.Ordinal)) + { + // Set an internal hashtag so that we can reuse changes in future snapshots. + options.Add(new KeyValuePair( + "hashtag", ComputeInternalHashTag(transformResult))); + } + + foreach (var r in LabelFinder.MapLabels(transformResult.GetLabelFinder(), _reviewersTemplate)) + { + options.Add(new KeyValuePair("r", r)); + } + foreach (var cc in LabelFinder.MapLabels(transformResult.GetLabelFinder(), _ccTemplate)) + { + options.Add(new KeyValuePair("cc", cc)); + } + foreach (var label in LabelFinder.MapLabels(transformResult.GetLabelFinder(), _labelsTemplate)) + { + options.Add(new KeyValuePair("label", label)); + } + + string result = components[0]; + if (result.StartsWith("refs/for/", StringComparison.Ordinal)) + { + string pushRef = result.Substring("refs/for/".Length); + if (_primaryBranchMigrationMode && (pushRef == "master" || pushRef == "main")) + { + string? primaryBranch = null; + try + { + primaryBranch = repo.GetPrimaryBranch(_repoUrl); + } + catch (RepoException e) + { + _console.WarnFmt("Unable to detect primary branch: {0}", e); + } + if (primaryBranch != null) + { + result = "refs/for/" + primaryBranch; + } + } + } + if (options.Count != 0) + { + result += "%" + string.Join(",", options.Select( + e => e.Key + (e.Value != null ? "=" + e.Value : ""))); + } + return result; + } + + public IReadOnlyList AfterPush( + string serverResponse, + GitDestination.MessageInfo messageInfo, + GitRevision pushedRevision, + IReadOnlyList originChanges) + { + // Should be the message info returned by generateMessageInfo. + var gerritMessageInfo = (GerritMessageInfo)messageInfo; + if (_gerritSubmit) + { + SubmitChange(gerritMessageInfo.ChangeId); + } + var originRefs = originChanges.Cast().ToList(); + var result = new List + { + new( + DestinationEffect.EffectType.CREATED, + $"Created revision {pushedRevision.GetHash()}", + originRefs, + new DestinationEffect.DestinationRef(pushedRevision.GetHash(), "commit", url: null)), + }; + + var lines = serverResponse.Split('\n'); + Match? gerritUrlMatcher = TryFindGerritUrl(lines); + if (gerritUrlMatcher == null || !gerritUrlMatcher.Success) + { + gerritUrlMatcher = TryFindGerritUrlOldFormat(lines); + } + if (gerritUrlMatcher != null && gerritUrlMatcher.Success) + { + string message = gerritMessageInfo.NewReview + ? "New Gerrit review created at " + : "Updated existing Gerrit review at "; + string url = gerritUrlMatcher.Groups[1].Value; + string changeNum = url.Substring(url.LastIndexOf('/') + 1); + message += url; + _console.Info(message); + if (_gerritSubmit) + { + message += ".\n Submited the change through API."; + } + result.Add( + new DestinationEffect( + gerritMessageInfo.NewReview + ? DestinationEffect.EffectType.CREATED + : DestinationEffect.EffectType.UPDATED, + message, + originRefs, + new DestinationEffect.DestinationRef(changeNum, "gerrit_review", url))); + } + + return result; + } + + private static Match? TryFindGerritUrl(IReadOnlyList lines) + { + bool successFound = false; + foreach (var line in lines) + { + if (line.Contains("SUCCESS")) + { + successFound = true; + } + if (successFound) + { + // Usually next line is empty, but best effort to find the URL after "SUCCESS". + Match urlMatcher = GerritUrlLine.Match(line); + if (urlMatcher.Success) + { + return urlMatcher; + } + } + } + return null; + } + + private static Match? TryFindGerritUrlOldFormat(IReadOnlyList lines) + { + for (int i = 0; i < lines.Count; i++) + { + string line = lines[i]; + if ((line.Contains("New Changes") || line.Contains("Updated Changes")) + && i + 1 < lines.Count) + { + Match urlMatcher = GerritUrlLine.Match(lines[i + 1]); + if (urlMatcher.Success) + { + return urlMatcher; + } + } + } + return null; + } + + private static string? GetExistingChangeId(string msg) + { + ChangeMessage changeMessage = ChangeMessage.ParseMessage(msg); + var labels = changeMessage.LabelsAsMultimap(); + if (labels.ContainsKey(ChangeIdLabel)) + { + var values = labels.Get(ChangeIdLabel); + return values[^1]; + } + return null; + } + + private GerritMessageInfo CreateMessageInfo( + TransformResult result, bool newReview, string gerritChangeId, ChangeIdPolicy changeIdPolicy) + { + IRevision rev = result.GetCurrentRevision(); + var labels = new List(); + if (result.IsSetRevId()) + { + labels.Add(new LabelFinder(result.GetRevIdLabel() + ": " + rev.AsString())); + } + string? existingChangeId = GetExistingChangeId(result.GetSummary()); + string? effectiveChangeId = existingChangeId; + switch (changeIdPolicy) + { + case ChangeIdPolicy.Require: + ValidationException.CheckCondition( + existingChangeId != null, + "{0} label not found in message:\n{1}", ChangeIdLabel, result.GetSummary()); + break; + case ChangeIdPolicy.FailIfPresent: + ValidationException.CheckCondition( + existingChangeId == null, + "{0} label found in message:\n{1}. You can use" + + " git.gerrit_destination(change_id_policy = ...) to change this behavior", + ChangeIdLabel, result.GetSummary()); + labels.Add(new LabelFinder(ChangeIdLabel + ": " + gerritChangeId)); + effectiveChangeId = gerritChangeId; + break; + case ChangeIdPolicy.Reuse: + if (existingChangeId == null) + { + labels.Add(new LabelFinder(ChangeIdLabel + ": " + gerritChangeId)); + effectiveChangeId = gerritChangeId; + } + break; + case ChangeIdPolicy.Replace: + labels.Add(new LabelFinder(ChangeIdLabel + ": " + gerritChangeId)); + effectiveChangeId = gerritChangeId; + break; + default: + throw new NotSupportedException("Unsupported policy: " + changeIdPolicy); + } + + return new GerritMessageInfo(labels, newReview, effectiveChangeId!); + } + + public IEndpoint GetFeedbackEndPoint(Console console) + { + _gerritOptions.ValidateEndpointChecker(_endpointChecker, _repoUrl); + return new GerritEndpoint( + _gerritOptions.NewGerritApiSupplier(_repoUrl, _endpointChecker), + _repoUrl, + console, + _gerritSubmit); + } + } + + public IDestination.IWriter NewWriter(WriterContext writerContext) => + _gitDestination.NewWriter(writerContext); + + public string GetLabelNameWhenOrigin() => GitRepository.GitOriginRevId; + + internal static GerritDestination NewGerritDestination( + GeneralOptions generalOptions, + GerritOptions gerritOptions, + GitOptions gitOptions, + GitDestinationOptions destinationOptions, + string url, + string fetch, + string pushToRefsFor, + bool submit, + bool partialFetch, + NotifyOption? notifyOption, + ChangeIdPolicy changeIdPolicy, + bool allowEmptyPatchSet, + IReadOnlyList reviewers, + IReadOnlyList cc, + IReadOnlyList labels, + IChecker? endpointChecker, + IEnumerable integrates, + string? topicTemplate, + bool gerritSubmit, + bool primaryBranchMigrationMode, + IChecker? checker, + CredentialFileHandler? credentials) + { + gerritSubmit = gerritOptions.ForceGerritSubmit ?? gerritSubmit; + submit = gerritOptions.ForceGerritSubmit ?? submit; + string push = submit && !gerritSubmit + ? pushToRefsFor + : $"refs/for/{pushToRefsFor}"; + return new GerritDestination( + new GitDestination( + url, + fetch, + push, + partialFetch, + primaryBranchMigrationMode, + tagName: null, + tagMsg: null, + destinationOptions, + gitOptions, + generalOptions, + new GerritWriteHook( + generalOptions, + gerritOptions, + url, + destinationOptions.GetCommitter(), + reviewers, + cc, + changeIdPolicy, + allowEmptyPatchSet, + labels, + endpointChecker, + notifyOption, + topicTemplate, + partialFetch, + gerritSubmit, + primaryBranchMigrationMode), + integrates, + checker, + credentials), + submit); + } + + public string GetType() => _submit ? _gitDestination.GetType() : "gerrit.destination"; + + public ImmutableListMultimap Describe(Glob? originFiles) + { + var builder = ImmutableListMultimap.CreateBuilder(); + builder.Put("gerritSubmit", _submit.ToString()); + if (_submit) + { + builder.PutAll(_gitDestination.Describe(originFiles)); + return builder.Build(); + } + foreach (var entry in _gitDestination.Describe(originFiles)) + { + if (entry.Key.Equals("type")) + { + continue; + } + builder.Put(entry.Key, entry.Value); + } + builder.Put("type", GetType()); + return builder.Build(); + } + + public IReadOnlyList> DescribeCredentials() => + _gitDestination.DescribeCredentials(); +} diff --git a/src/Copybara.Core/Git/GerritEndpoint.cs b/src/Copybara.Core/Git/GerritEndpoint.cs new file mode 100644 index 000000000..b35afde23 --- /dev/null +++ b/src/Copybara.Core/Git/GerritEndpoint.cs @@ -0,0 +1,299 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text; +using Copybara.Common; +using Copybara.Config; +using Copybara.Exceptions; +using Copybara.Git.GerritApi; +using Starlark.Annot; +using Starlark.Eval; +using Console = Copybara.Util.Console.Console; +using GerritApiClient = Copybara.Git.GerritApi.GerritApi; + +namespace Copybara.Git; + +/// +/// Gerrit endpoint implementation for feedback migrations. Port of +/// com.google.copybara.git.GerritEndpoint. +/// +[StarlarkBuiltin( + "gerrit_api_obj", + Doc = "Gerrit API endpoint implementation for feedback migrations and after migration hooks.")] +public sealed class GerritEndpoint : IEndpoint, IStarlarkValue +{ + private const int GerritMaxMessageBytes = 16 << 10; + private const string TruncatedPrefix = "(truncated): "; + private static readonly int TruncatedPrefixBytes = Encoding.UTF8.GetByteCount(TruncatedPrefix); + private static readonly int TruncatedMessageMaxBytes = + TruncatedPrefixBytes + GerritMaxMessageBytes; + + private readonly LazyResourceLoader _apiSupplier; + private readonly string _url; + private readonly Console _console; + private readonly bool _allowSubmitChange; + + internal GerritEndpoint( + LazyResourceLoader apiSupplier, + string url, + Console console, + bool allowSubmitChange) + { + _apiSupplier = Preconditions.CheckNotNull(apiSupplier); + _url = Preconditions.CheckNotNull(url); + _console = Preconditions.CheckNotNull(console); + _allowSubmitChange = allowSubmitChange; + } + + [StarlarkMethod("get_change", Doc = "Retrieve a Gerrit change.")] + public ChangeInfo GetChange( + [Param(Name = "id", Named = true, Doc = "The change id or change number.")] string id, + [Param( + Name = "include_results", + Named = true, + Doc = "What to include in the response.", + Positional = false, + DefaultValue = "['LABELS']")] + ISequence includeResults) + { + try + { + ChangeInfo changeInfo = DoGetChange(id, GetIncludeResults(includeResults)); + ValidationException.CheckCondition( + !changeInfo.IsMoreChanges(), "Pagination is not supported yet."); + return changeInfo; + } + catch (GerritApiException re) + { + throw HandleGerritApiException(re, "get_change"); + } + catch (Exception e) when (e is RepoException or ValidationException) + { + throw new EvalException("Error getting change: " + e.Message, e); + } + } + + [StarlarkMethod("get_actions", Doc = "Retrieve the actions of a Gerrit change.")] + public IReadOnlyDictionary GetActions( + [Param(Name = "id", Named = true, Doc = "The change id or change number.")] string id, + [Param(Name = "revision", Named = true, Doc = "The revision of the change.")] string revision) + { + try + { + GerritApiClient gerritApi = _apiSupplier.Load(_console); + return gerritApi.GetActionsAsync(id, revision).GetAwaiter().GetResult(); + } + catch (GerritApiException re) + { + throw HandleGerritApiException(re, "get_actions"); + } + catch (Exception e) when (e is RepoException or ValidationException) + { + throw new EvalException("Error getting actions: " + e.Message, e); + } + } + + private static IReadOnlySet GetIncludeResults(ISequence includeResults) + { + var enumResults = new HashSet(); + foreach (var result in includeResults) + { + enumResults.Add( + SkylarkUtil.StringToEnum("include_results", (string)result!)); + } + return enumResults; + } + + private ChangeInfo DoGetChange(string changeId, IReadOnlySet includeResults) + { + GerritApiClient gerritApi = _apiSupplier.Load(_console); + return gerritApi.GetChangeAsync(changeId, new GetChangeInput(includeResults)) + .GetAwaiter().GetResult(); + } + + private static ValidationException HandleGerritApiException( + GerritApiException re, string methodName) + { + int responseCode = (int)re.GetResponseCode(); + if (responseCode is >= 400 and < 500) + { + return new ValidationException( + $"Request error calling {methodName}. Gerrit returned a request error while" + + $" attempting to post a review:\n{re.Message}", + re); + } + return new ValidationException("Error calling " + methodName, re); + } + + [StarlarkMethod( + "post_review", + Doc = + "Post a review to a Gerrit change for a particular revision. The review will be authored " + + "by the user running the tool, or the role account if running in the service.\n")] + public ReviewResult PostReview( + [Param(Name = "change_id", Named = true, Doc = "The Gerrit change id.")] string changeId, + [Param( + Name = "revision_id", + Named = true, + Doc = "The revision for which the comment will be posted.")] + string revisionId, + [Param(Name = "review_input", Named = true, Doc = "The review to post to Gerrit.")] + SetReviewInput reviewInput) + { + SetReviewInput finalReviewInput = MaybeTruncateMessage(reviewInput); + try + { + GerritApiClient gerritApi = _apiSupplier.Load(_console); + return gerritApi.SetReviewAsync(changeId, revisionId, finalReviewInput) + .GetAwaiter().GetResult(); + } + catch (GerritApiException re) + { + throw HandleGerritApiException(re, "post_review"); + } + catch (Exception e) when (e is RepoException or ValidationException) + { + throw new EvalException("Error calling post_review: " + e.Message, e); + } + } + + private static SetReviewInput MaybeTruncateMessage(SetReviewInput reviewInput) + { + if (reviewInput.GetMessage() != null + && Encoding.UTF8.GetByteCount(reviewInput.GetMessage()!) > GerritMaxMessageBytes) + { + string nonTruncatedMessage = TruncatedPrefix + reviewInput.GetMessage(); + // Assume each char is largest case scenario of 4 bytes. + string truncatedMessage = nonTruncatedMessage.Substring(0, TruncatedMessageMaxBytes / 4); + return SetReviewInput.Create( + truncatedMessage, + reviewInput.GetLabels(), + reviewInput.GetTag(), + reviewInput.GetNotify() ?? NotifyType.ALL); + } + return reviewInput; + } + + [StarlarkMethod( + "delete_vote", + Doc = "Delete a label vote from an account owner on a Gerrit change.\n")] + public void DeleteVote( + [Param(Name = "change_id", Named = true, Doc = "The Gerrit change id.")] string changeId, + [Param( + Name = "account_id", + Named = true, + Doc = "The account owner who votes on label_id. Use 'me' or 'self' if the account owner" + + " makes this api call")] + string accountId, + [Param(Name = "label_id", Named = true, Doc = "The name of the label.")] string labelId) + { + try + { + GerritApiClient gerritApi = _apiSupplier.Load(_console); + gerritApi.DeleteVoteAsync(changeId, accountId, labelId, new DeleteVoteInput(NotifyType.NONE)) + .GetAwaiter().GetResult(); + } + catch (GerritApiException re) + { + throw HandleGerritApiException(re, "delete_vote"); + } + catch (Exception e) when (e is RepoException or ValidationException) + { + throw new EvalException("Error calling delete_vote: " + e.Message, e); + } + } + + [StarlarkMethod("submit_change", Doc = "Submit a Gerrit change")] + public ChangeInfo SubmitChange( + [Param(Name = "change_id", Named = true, Doc = "The Gerrit change id.")] string changeId) + { + ValidationException.CheckCondition( + _allowSubmitChange, + "Gerrit submit_change is only allowed if it is is enabled on the endpoint"); + try + { + GerritApiClient gerritApi = _apiSupplier.Load(_console); + return gerritApi.SubmitChangeAsync(changeId, new SubmitInput(NotifyType.NONE)) + .GetAwaiter().GetResult(); + } + catch (GerritApiException re) + { + throw HandleGerritApiException(re, "submit_change"); + } + catch (Exception e) when (e is RepoException or ValidationException) + { + throw new EvalException("Error calling submit_change: " + e.Message, e); + } + } + + [StarlarkMethod("abandon_change", Doc = "Abandon a Gerrit change.")] + public ChangeInfo AbandonChange( + [Param(Name = "change_id", Named = true, Doc = "The Gerrit change id.")] string changeId) + { + try + { + GerritApiClient gerritApi = _apiSupplier.Load(_console); + return gerritApi.AbandonChangeAsync(changeId, AbandonInput.CreateWithoutComment()) + .GetAwaiter().GetResult(); + } + catch (GerritApiException re) + { + throw HandleGerritApiException(re, "abandon_change"); + } + catch (Exception e) when (e is RepoException or ValidationException) + { + throw new EvalException("Error getting change: " + e.Message, e); + } + } + + [StarlarkMethod( + "list_changes", + Doc = "Get changes from Gerrit based on a query.\n")] + public StarlarkList ListChanges( + [Param(Name = "query", Named = true, Doc = "The query string to list changes by.")] + string queryString, + [Param( + Name = "include_results", + Named = true, + Doc = "What to include in the response.", + Positional = false, + DefaultValue = "[]")] + ISequence includeResults) + { + GerritApiClient gerritApi = _apiSupplier.Load(_console); + var changes = gerritApi.GetChangesAsync( + new ChangesQuery(queryString).WithInclude(GetIncludeResults(includeResults))) + .GetAwaiter().GetResult(); + return StarlarkList.ImmutableCopyOf(changes); + } + + [StarlarkMethod("url", Doc = "Return the URL of this endpoint.", StructField = true)] + public string GetUrl() => _url; + + public IEndpoint WithConsole(Console console) => + new GerritEndpoint(_apiSupplier, _url, console, _allowSubmitChange); + + public ImmutableListMultimap Describe() + { + var builder = ImmutableListMultimap.CreateBuilder(); + builder.Put("type", "gerrit_api"); + builder.Put("url", _url); + builder.Put("gerritSubmit", _allowSubmitChange.ToString()); + return builder.Build(); + } + + public override string ToString() => $"GerritEndpoint{{url={_url}}}"; +} diff --git a/src/Copybara.Core/Git/GerritEventTrigger.cs b/src/Copybara.Core/Git/GerritEventTrigger.cs new file mode 100644 index 000000000..d4efb733b --- /dev/null +++ b/src/Copybara.Core/Git/GerritEventTrigger.cs @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2022 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Git.GerritApi; + +namespace Copybara.Git; + +/// +/// A simple pair to express Gerrit Events with arbitrary subtypes (Labels). Port of +/// com.google.copybara.git.GerritEventTrigger. +/// +public sealed class GerritEventTrigger +{ + private GerritEventTrigger(GerritEventType type, IReadOnlyList subtypes) + { + Type = type; + Subtypes = subtypes; + } + + public GerritEventType Type { get; } + + public IReadOnlyList Subtypes { get; } + + public static GerritEventTrigger Create(GerritEventType type, IEnumerable subtypes) => + new(type, subtypes.ToImmutableArray()); + + public override string ToString() => Type.ToString(); +} diff --git a/src/Copybara.Core/Git/GerritIntegrateLabel.cs b/src/Copybara.Core/Git/GerritIntegrateLabel.cs new file mode 100644 index 000000000..7504b54e5 --- /dev/null +++ b/src/Copybara.Core/Git/GerritIntegrateLabel.cs @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.RegularExpressions; +using Copybara.Common; + +namespace Copybara.Git; + +/// +/// Integrate label for Gerrit changes. Port of +/// com.google.copybara.git.GerritIntegrateLabel. +/// +/// Returns a string like: +/// +/// "Gerrit https://example.com/project 1271" +/// "Gerrit https://example.com/project 1271 5" +/// "Gerrit https://example.com/project 1271 ChangeId" +/// "Gerrit https://example.com/project 1271 5 ChangeId" +/// +/// Where both the PatchSet and ChangeId are optional. +/// +internal sealed class GerritIntegrateLabel : IIntegrateLabel +{ + private static readonly Regex LabelPattern = + new("^gerrit ([^ ]+) ([0-9]+)(?: Patch Set ([0-9]+))?(?: (I[a-f0-9]+))?$", + RegexOptions.Compiled); + + private readonly GitRepository _repository; + private readonly GeneralOptions _generalOptions; + private readonly string _url; + private readonly int _changeNumber; + private int? _patchSet; + private readonly string? _changeId; + + internal GerritIntegrateLabel( + GitRepository repository, + GeneralOptions generalOptions, + string url, + int changeNumber, + int? patchSet, + string? changeId) + { + _repository = Preconditions.CheckNotNull(repository); + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _url = Preconditions.CheckNotNull(url); + _changeNumber = changeNumber; + _patchSet = patchSet; + _changeId = changeId; + } + + internal static GerritIntegrateLabel? Parse( + string str, GitRepository repository, GeneralOptions generalOptions) + { + Match matcher = LabelPattern.Match(str.Trim()); + return matcher.Success + ? new GerritIntegrateLabel( + repository, + generalOptions, + matcher.Groups[1].Value, + int.Parse(matcher.Groups[2].Value), + matcher.Groups[3].Success && matcher.Groups[3].Value.Length > 0 + ? int.Parse(matcher.Groups[3].Value) + : null, + matcher.Groups[4].Success && matcher.Groups[4].Value.Length > 0 + ? matcher.Groups[4].Value + : null) + : null; + } + + public override string ToString() => + string.Format( + "gerrit {0} {1}{2}{3}", + _url, + _changeNumber, + _patchSet != null ? " Patch Set " + _patchSet : "", + _changeId != null ? " " + _changeId : ""); + + public string MergeMessage(IReadOnlyList labelsToAdd) + { + if (_changeId != null) + { + var updated = new List(labelsToAdd) + { + new("Change-Id: " + _changeId), + }; + labelsToAdd = updated; + } + return IIntegrateLabel.WithLabels( + "Merge Gerrit change " + _changeNumber + + (_patchSet == null ? "" : " Patch Set " + _patchSet), + labelsToAdd); + } + + public GitRevision GetRevision() + { + var patchSets = GerritChange.GetGerritPatchSets(_repository, _url, _changeNumber); + int latestPatchSet = patchSets.Keys.Last(); + + if (_patchSet == null) + { + _patchSet = latestPatchSet; + } + else if (latestPatchSet > _patchSet) + { + _generalOptions.GetConsole().WarnFmt( + "Change {0} has more patch sets after Patch Set {1}. Latest is Patch Set {2}." + + " Not all changes might be migrated", + _changeNumber, _patchSet, latestPatchSet); + } + + return GitRepoType.Gerrit.ResolveRef( + _repository, + _url, + $"refs/changes/{_changeNumber % 100:D2}/{_changeNumber}" + "/" + _patchSet, + _generalOptions, + describeVersion: false, + partialFetch: false, + fetchDepth: null); + } +} diff --git a/src/Copybara.Core/Git/GerritOptions.cs b/src/Copybara.Core/Git/GerritOptions.cs new file mode 100644 index 000000000..5b5f2bc63 --- /dev/null +++ b/src/Copybara.Core/Git/GerritOptions.cs @@ -0,0 +1,184 @@ +/* + * Copyright (C) 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.RegularExpressions; +using Copybara.Checks; +using Copybara.Exceptions; +using Copybara.Git.GerritApi; +using Copybara.Http; +using Console = Copybara.Util.Console.Console; +using GerritApiClient = Copybara.Git.GerritApi.GerritApi; +using ProfilerType = Copybara.Profiler.Profiler; + +namespace Copybara.Git; + +/// +/// Arguments for . Port of +/// com.google.copybara.git.GerritOptions. +/// +public class GerritOptions : IOption +{ + private static readonly Regex ChangeIdPattern = new("^I[0-9a-f]{40}$", RegexOptions.Compiled); + + protected readonly GeneralOptions GeneralOptions; + protected GitOptions GitOptions; + private ISleeper _sleeper = SystemSleeper.Instance; + + // The HttpOptions are used to obtain an HttpClient. Java uses NetHttpTransport directly. + private readonly HttpOptions _httpOptions; + + public GerritOptions(GeneralOptions generalOptions, GitOptions gitOptions) + : this(generalOptions, gitOptions, new HttpOptions()) + { + } + + public GerritOptions(GeneralOptions generalOptions, GitOptions gitOptions, HttpOptions httpOptions) + { + GeneralOptions = generalOptions; + GitOptions = gitOptions; + _httpOptions = httpOptions; + } + + public ISleeper GetSleeper() => _sleeper; + + public void SetSleeper(ISleeper sleeper) => _sleeper = sleeper; + + // --gerrit-change-id + public string GerritChangeId { get; set; } = ""; + + // --gerrit-new-change + public bool NewChange { get; set; } + + // --gerrit-topic + public string GerritTopic { get; set; } = ""; + + // --force-gerrit-submit + public bool? ForceGerritSubmit { get; set; } + + // --nogerrit-rev-id-label (DEPRECATED) + public bool NoRevIdDeprecated { get; set; } + + /// Validate that the argument is a valid Gerrit Change-id. + public static void ValidateChangeId(string name, string value) + { + if (!string.IsNullOrEmpty(value) && !ChangeIdPattern.IsMatch(value)) + { + throw new ArgumentException( + $"{name} value '{value}' does not match Gerrit Change ID pattern: {ChangeIdPattern}"); + } + } + + /// Returns a lazy supplier of . + internal LazyResourceLoader NewGerritApiSupplier(string url, IChecker? checker) => + LazyResourceLoader.Memoized(console => + checker == null + ? NewGerritApi(url) + : NewGerritApi(url, checker, console!)); + + /// Override this method in a class for a specific Gerrit implementation. + public virtual GerritApiClient NewGerritApi(string url) => NewGerritApi(url, null, null); + + /// Creates a new enforcing the given checker. + protected virtual GerritApiClient NewGerritApi(string url, IChecker? checker, Console? console) + { + if (checker == null) + { + return new GerritApiClient(NewGerritApiTransport(HostUrl(url)), GeneralOptions.Profiler()); + } + return new GerritApiClient( + NewGerritApiTransport(HostUrl(url), checker, console!), GeneralOptions.Profiler()); + } + + /// Return the url removing the path part, since the API needs the host. + protected static Uri HostUrl(string url) + { + Uri result = AsUri(url); + ValidationException.CheckCondition(result.Host != null, "Wrong url: {0}", url); + ValidationException.CheckCondition(result.Scheme != null, "Wrong url: {0}", url); + var builder = new UriBuilder(result.Scheme, result.Host, result.IsDefaultPort ? -1 : result.Port) + { + UserName = result.UserInfo, + Path = string.Empty, + Query = string.Empty, + Fragment = string.Empty, + }; + return builder.Uri; + } + + private static Uri AsUri(string url) + { + try + { + return new Uri(url); + } + catch (UriFormatException) + { + throw new ValidationException("Invalid URL " + url); + } + } + + /// + /// Given a repo url, return the project part. + /// + /// Not static on purpose, since we might introduce different behavior based on other flags + /// in the future. + /// + public string GetProject(string url) + { + string file = AsUri(url).AbsolutePath; + if (file.StartsWith('/')) + { + file = file.Substring(1); + } + if (file.EndsWith('/')) + { + file = file.Substring(0, file.Length - 1); + } + return Regex.Replace(file, "[ \"'&]", ""); + } + + /// Create a Gerrit http transport for a URI. + protected virtual IGerritApiTransport NewGerritApiTransport(Uri uri) => + new GerritApiTransportImpl(GetCredentialsRepo(), uri, _httpOptions.GetTransport()); + + /// Create a Gerrit http transport for a URI and checker. + protected virtual IGerritApiTransport NewGerritApiTransport( + Uri uri, IChecker checker, Console console) => + new GerritApiTransportWithChecker(NewGerritApiTransport(uri), checker, console); + + protected virtual GitRepository GetCredentialsRepo() => + GitOptions.CachedBareRepoForUrl("just_for_github_api"); + + /// Validate if a checker is valid to use with a Gerrit endpoint for repoUrl. + public virtual void ValidateEndpointChecker(IChecker? checker, string repoUrl) + { + // Accept any by default + } +} + +/// Abstraction over Thread.Sleep to allow overriding in tests. Port of TestSleeper. +public interface ISleeper +{ + void Sleep(long millis); +} + +/// The system sleeper implementation. +public sealed class SystemSleeper : ISleeper +{ + public static readonly SystemSleeper Instance = new(); + + public void Sleep(long millis) => Thread.Sleep((int)millis); +} diff --git a/src/Copybara.Core/Git/GerritOrigin.cs b/src/Copybara.Core/Git/GerritOrigin.cs new file mode 100644 index 000000000..68177da61 --- /dev/null +++ b/src/Copybara.Core/Git/GerritOrigin.cs @@ -0,0 +1,354 @@ +/* + * Copyright (C) 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Approval; +using Copybara.Checks; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Git.GerritApi; +using Copybara.Revision; +using Copybara.Util; +using Console = Copybara.Util.Console.Console; +using GerritApiClient = Copybara.Git.GerritApi.GerritApi; + +namespace Copybara.Git; + +/// +/// An that can read Gerrit reviews. Port of +/// com.google.copybara.git.GerritOrigin. +/// +public sealed class GerritOrigin : GitOrigin +{ + private readonly GeneralOptions _generalOptions; + private readonly GitOptions _gitOptions; + private readonly GitOriginOptions _gitOriginOptions; + private readonly GerritOptions _gerritOptions; + private readonly GitOrigin.SubmoduleStrategy _submoduleStrategy; + private readonly IReadOnlyList _excludedSubmodules; + private readonly bool _includeBranchCommitLogs; + private readonly bool _partialFetch; + private readonly IChecker? _endpointChecker; + private readonly ITransformation? _patchTransformation; + private readonly string? _branch; + private readonly bool _ignoreGerritNoop; + private readonly bool _importWipChanges; + + private GerritOrigin( + GeneralOptions generalOptions, + string repoUrl, + string? configRef, + GitOptions gitOptions, + GitOriginOptions gitOriginOptions, + GerritOptions gerritOptions, + GitOrigin.SubmoduleStrategy submoduleStrategy, + IReadOnlyList excludedSubmodules, + bool includeBranchCommitLogs, + bool firstParent, + bool partialFetch, + IChecker? endpointChecker, + ITransformation? patchTransformation, + string? branch, + bool describeVersion, + bool ignoreGerritNoop, + bool primaryBranchMigrationMode, + IApprovalsProvider approvalsProvider, + bool importWipChanges, + IGitRepositoryHook? gitRepositoryHook) + : base( + generalOptions, + repoUrl, + configRef, + GitRepoType.Gerrit, + gitOptions, + gitOriginOptions, + submoduleStrategy, + excludedSubmodules, + includeBranchCommitLogs, + firstParent, + partialFetch, + patchTransformation, + describeVersion, + versionSelector: null, + configPath: null, + workflowName: null, + primaryBranchMigrationMode, + approvalsProvider, + enableLfs: false, + credentials: null, + gitRepositoryHook) + { + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _gitOptions = Preconditions.CheckNotNull(gitOptions); + _gitOriginOptions = Preconditions.CheckNotNull(gitOriginOptions); + _gerritOptions = Preconditions.CheckNotNull(gerritOptions); + _submoduleStrategy = submoduleStrategy; + _excludedSubmodules = excludedSubmodules; + _includeBranchCommitLogs = includeBranchCommitLogs; + _endpointChecker = endpointChecker; + _patchTransformation = patchTransformation; + _branch = branch; + _partialFetch = partialFetch; + _ignoreGerritNoop = ignoreGerritNoop; + _importWipChanges = importWipChanges; + } + + public override ImmutableListMultimap Describe(Glob? originFiles) + { + var builder = ImmutableListMultimap.CreateBuilder(); + builder.PutAll(base.Describe(originFiles)); + if (_branch != null) + { + builder.Put("branch", _branch); + } + builder.Put("import_wip_changes", _importWipChanges.ToString()); + return builder.Build(); + } + + public override GitRevision Resolve(string? reference) + { + _generalOptions.GetConsole().Progress("Gerrit Origin: Initializing local repo"); + + ValidationException.CheckCondition( + !string.IsNullOrEmpty(reference), "Expecting a change number as reference"); + + GerritChange? change = GerritChange.Resolve(GetRepository(), RepoUrl, reference!, _generalOptions); + if (change == null) + { + GitRevision gitRevisionResolved = GitRepoType.Git.ResolveRef( + GetRepository(), RepoUrl, reference!, _generalOptions, DescribeVersion, _partialFetch, + fetchDepth: null); + return DescribeVersion + ? GetRepository().AddDescribeVersion(gitRevisionResolved) + : gitRevisionResolved; + } + GerritApiClient api = _gerritOptions.NewGerritApi(RepoUrl); + + ChangeInfo response = api.GetChangeAsync( + change.GetChange().ToString(), + new GetChangeInput( + new HashSet + { + IncludeResult.DETAILED_ACCOUNTS, + IncludeResult.DETAILED_LABELS, + })) + .GetAwaiter().GetResult(); + + if (_branch != null && !_branch.Equals(response.GetBranch())) + { + throw new EmptyChangeException( + $"Skipping import of change {change.GetChange()} for branch {response.GetBranch()}." + + $" Only tracking changes for branch {_branch}"); + } + + if (!_importWipChanges && response.IsWorkInProgress()) + { + throw new EmptyChangeException( + $"Skipping import of change {change.GetChange()} as it is marked as Work in Progress."); + } + + var labels = ImmutableListMultimap.CreateBuilder(); + + labels.Put(GerritChange.GerritChangeBranch, response.GetBranch()!); + if (response.GetTopic() != null) + { + labels.Put(GerritChange.GerritChangeTopic, response.GetTopic()!); + } + labels.Put(GerritChange.GerritCompleteChangeIdLabel, response.GetId()!); + foreach (var e in response.GetReviewers()) + { + foreach (var info in e.Value) + { + if (info.GetEmail() != null) + { + labels.Put("GERRIT_" + e.Key + "_EMAIL", info.GetEmail()!); + } + } + } + + if (response.GetOwner()?.GetEmail() != null) + { + labels.Put(GerritChange.GerritOwnerEmailLabel, response.GetOwner()!.GetEmail()!); + } + try + { + GitRevision gitRevision = change.Fetch(labels.Build()); + return DescribeVersion ? GetRepository().AddDescribeVersion(gitRevision) : gitRevision; + } + catch (CannotResolveRevisionException unexpected) + { + // We got the change via the API so it is unexpected to fail now. + throw new RepoException("Unable to fetch change content.", unexpected); + } + } + + /// Builds a new . + internal static GerritOrigin NewGerritOrigin( + GeneralOptions generalOptions, + GitOptions gitOptions, + GitOriginOptions gitOriginOptions, + GerritOptions gerritOptions, + GitDestinationOptions destinationOptions, + string url, + GitOrigin.SubmoduleStrategy submoduleStrategy, + IReadOnlyList excludedSubmodules, + bool firstParent, + bool partialFetch, + IChecker? endpointChecker, + ITransformation? patchTransformation, + string? branch, + bool describeVersion, + bool ignoreGerritNoop, + bool primaryBranchMigrationMode, + IApprovalsProvider approvalsProvider, + bool importWipChanges, + IGitRepositoryHook? gitRepositoryHook) => + new( + generalOptions, + url, + configRef: null, + gitOptions, + gitOriginOptions, + gerritOptions, + submoduleStrategy, + excludedSubmodules, + includeBranchCommitLogs: false, + firstParent, + partialFetch, + endpointChecker, + patchTransformation, + branch, + describeVersion, + ignoreGerritNoop, + primaryBranchMigrationMode, + approvalsProvider, + importWipChanges, + gitRepositoryHook); + + public override IOrigin.IReader NewReader( + Glob originFiles, Authoring.Authoring authoring) => + new GerritReaderImpl( + RepoUrl, + originFiles, + authoring, + _gitOptions, + _gitOriginOptions, + _generalOptions, + _includeBranchCommitLogs, + _submoduleStrategy, + _excludedSubmodules, + FirstParent, + _partialFetch, + _patchTransformation, + configPath: null, + workflowName: null, + credentials: null, + GitRepositoryHook, + _gerritOptions, + _endpointChecker, + _ignoreGerritNoop); + + private sealed class GerritReaderImpl : ReaderImpl + { + private readonly GerritOptions _gerritOptions; + private readonly IChecker? _endpointChecker; + private readonly bool _ignoreGerritNoop; + + internal GerritReaderImpl( + string repoUrl, + Glob originFiles, + Authoring.Authoring authoring, + GitOptions gitOptions, + GitOriginOptions gitOriginOptions, + GeneralOptions generalOptions, + bool includeBranchCommitLogs, + GitOrigin.SubmoduleStrategy submoduleStrategy, + IReadOnlyList excludedSubmodules, + bool firstParent, + bool partialFetch, + ITransformation? patchTransformation, + string? configPath, + string? workflowName, + CredentialFileHandler? credentials, + IGitRepositoryHook? gitRepositoryHook, + GerritOptions gerritOptions, + IChecker? endpointChecker, + bool ignoreGerritNoop) + : base( + repoUrl, + originFiles, + authoring, + gitOptions, + gitOriginOptions, + generalOptions, + includeBranchCommitLogs, + submoduleStrategy, + excludedSubmodules, + firstParent, + partialFetch, + patchTransformation, + configPath, + workflowName, + credentials, + gitRepositoryHook) + { + _gerritOptions = gerritOptions; + _endpointChecker = endpointChecker; + _ignoreGerritNoop = ignoreGerritNoop; + } + + public override IReadOnlyList FindBaselinesWithoutLabel( + GitRevision startRevision, int limit) + { + // Skip the initial change as it might be the Gerrit review change. + var visitor = new BaselinesWithoutLabelVisitor( + OriginFiles, limit, startRevision, skipFirst: false); + VisitChanges(startRevision, visitor); + return visitor.GetResult(); + } + + public override IEndpoint GetFeedbackEndPoint(Console console) + { + _gerritOptions.ValidateEndpointChecker(_endpointChecker, RepoUrl); + return new GerritEndpoint( + _gerritOptions.NewGerritApiSupplier(RepoUrl, _endpointChecker), + RepoUrl, + console, + // We disallow submitting to the origin, but this has feasible use cases and we can + // revisit. + allowSubmitChange: false); + } + + public override Origin.ChangesResponse Changes( + GitRevision? fromRef, GitRevision toRef) + { + Origin.ChangesResponse result = base.Changes(fromRef, toRef); + Change change = Change(toRef); + if (!_ignoreGerritNoop + || change.GetChangeFiles() == null + || !toRef.AssociatedLabels().ContainsKey(GerritChange.GerritCompleteChangeIdLabel)) + { + return result; + } + var pathMatcher = OriginFiles.RelativeTo("/"); + if (!change.GetChangeFiles()!.Any(x => pathMatcher.Matches("/" + x))) + { + return Origin.ChangesResponse.NoChanges(Origin.EmptyReason.NoChanges); + } + return result; + } + } +} diff --git a/src/Copybara.Core/Git/GerritTrigger.cs b/src/Copybara.Core/Git/GerritTrigger.cs new file mode 100644 index 000000000..a27dd3f43 --- /dev/null +++ b/src/Copybara.Core/Git/GerritTrigger.cs @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; +using Console = Copybara.Util.Console.Console; +using GerritApiClient = Copybara.Git.GerritApi.GerritApi; + +namespace Copybara.Git; + +/// +/// A feedback trigger based on updates on a Gerrit change. Port of +/// com.google.copybara.git.GerritTrigger. +/// +public sealed class GerritTrigger : ITrigger +{ + private readonly LazyResourceLoader _apiSupplier; + private readonly string _url; + private readonly IReadOnlySet _events; + private readonly Console _console; + private readonly bool _allowSubmitChange; + + internal GerritTrigger( + LazyResourceLoader apiSupplier, + string url, + IReadOnlySet events, + Console console, + bool allowSubmitChange) + { + _apiSupplier = Preconditions.CheckNotNull(apiSupplier); + _url = Preconditions.CheckNotNull(url); + _events = Preconditions.CheckNotNull(events); + _console = console; + _allowSubmitChange = allowSubmitChange; + } + + public IEndpoint GetEndpoint() => + new GerritEndpoint(_apiSupplier, _url, _console, _allowSubmitChange); + + public ImmutableListMultimap Describe() + { + var builder = ImmutableListMultimap.CreateBuilder(); + builder.Put("type", "gerrit_trigger"); + builder.Put("url", _url); + builder.Put("gerritSubmit", _allowSubmitChange.ToString()); + builder.PutAll("events", _events.Select(s => s.Type.ToString())); + + foreach (var trigger in _events) + { + if (trigger.Subtypes.Count == 0) + { + continue; + } + builder.PutAll($"SUBTYPES_{trigger.Type}", trigger.Subtypes); + } + return builder.Build(); + } + + public override string ToString() => + $"GerritTrigger{{url={_url}, event_types={string.Join(",", _events)}}}"; +} diff --git a/src/Copybara.Core/Git/GitCredential.cs b/src/Copybara.Core/Git/GitCredential.cs new file mode 100644 index 000000000..2225d9859 --- /dev/null +++ b/src/Copybara.Core/Git/GitCredential.cs @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text; +using System.Text.RegularExpressions; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Util; + +namespace Copybara.Git; + +/// +/// Utility class for executing 'git credential' commands. Port of +/// com.google.copybara.git.GitCredential. +/// +public sealed class GitCredential +{ + private static readonly Regex NewLine = new(@"\r\n|\n|\r", RegexOptions.Compiled); + + private readonly TimeSpan _timeout; + private readonly GitEnvironment _gitEnv; + + public GitCredential(TimeSpan timeout, GitEnvironment gitEnv) + { + _timeout = timeout; + _gitEnv = Preconditions.CheckNotNull(gitEnv); + } + + /// + /// Execute 'git credential fill' for a url. + /// + /// the directory to execute the command in. This is important if credential + /// configuration is set in the local git config. + /// url to get the credentials from + /// a username and password + /// If the url doesn't have a protocol, is not valid, or the + /// username/password couldn't be found. + public UserPassword Fill(string gitDir, string url) + { + var env = _gitEnv.WithNoGitPrompt().GetEnvironment(); + + Uri uri; + try + { + uri = new Uri(url); + } + catch (Exception e) when (e is UriFormatException or InvalidOperationException) + { + throw new ValidationException("Cannot get credentials for " + url, e); + } + string? protocol = uri.Scheme; + ValidationException.CheckCondition( + !string.IsNullOrEmpty(protocol), "Cannot find the protocol for %s", url); + string host = uri.Host; + + var cmd = new Command( + new[] { _gitEnv.ResolveGitBinary(), "--git-dir=" + gitDir, "credential", "fill" }, + env, + gitDir); + + var request = new StringBuilder(); + request.Append($"protocol={protocol}\nhost={host}\n"); + if (!string.IsNullOrEmpty(uri.AbsolutePath)) + { + request.Append($"path={uri.AbsolutePath.TrimStart('/')}\n"); + } + request.Append('\n'); + + CommandOutputWithStatus result; + try + { + // DON'T LOG THE OUTPUT. WE DON'T WANT TO ACCIDENTALLY LOG THE PASSWORD! + result = new CommandRunner(cmd, _timeout) + .WithMaxStdOutLogLines(0) + .WithInput(Encoding.UTF8.GetBytes(request.ToString())) + .Execute(); + } + catch (BadExitStatusWithOutputException e) + { + string errStr = e.GetOutput().GetStderr(); + ValidationException.CheckCondition( + !errStr.Contains("could not read"), + "Interactive prompting of passwords for git is disabled," + + " use git credential store before calling Copybara."); + throw new RepoException("Error getting credentials:\n" + errStr, e); + } + catch (CommandException e) + { + throw new RepoException("Error getting credentials", e); + } + + var map = new Dictionary(); + foreach (var line in NewLine.Split(result.GetStdout())) + { + if (line.Length == 0) + { + continue; + } + int idx = line.IndexOf('='); + if (idx < 0) + { + continue; + } + map[line.Substring(0, idx)] = line.Substring(idx + 1); + } + + if (!map.TryGetValue("username", out var username)) + { + throw new RepoException("git credentials for " + url + " didn't return a username"); + } + if (!map.TryGetValue("password", out var password)) + { + throw new RepoException("git credentials for " + url + " didn't return a password"); + } + return new UserPassword(username, password); + } + + /// A class that contains a username and password for git repositories. + public sealed class UserPassword + { + private readonly string _username; + private readonly string _password; + + internal UserPassword(string username, string password) + { + _username = Preconditions.CheckNotNull(username); + _password = Preconditions.CheckNotNull(password); + } + + // DON'T CHANGE THIS: never expose the password in ToString. + public override string ToString() => + $"UserPassword{{username={_username}, password=(hidden)}}"; + + public string GetUsername() => _username; + + /// Get the password. BE CAREFUL AND DON'T LOG IT! + public string GetPasswordBeCareful() => _password; + } +} diff --git a/src/Copybara.Core/Git/GitDestination.cs b/src/Copybara.Core/Git/GitDestination.cs new file mode 100644 index 000000000..eb920bdd2 --- /dev/null +++ b/src/Copybara.Core/Git/GitDestination.cs @@ -0,0 +1,1000 @@ +/* + * Copyright (C) 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Checks; +using Copybara.Common; +using Copybara.Effect; +using Copybara.Exceptions; +using Copybara.Revision; +using Copybara.Util; +using Starlark.Annot; +using Starlark.Eval; +using Console = Copybara.Util.Console.Console; + +namespace Copybara.Git; + +/// A Git repository destination. Port of com.google.copybara.git.GitDestination. +public class GitDestination : IDestination +{ + private const string OriginLabelSeparator = ": "; + public const int SmallNumFilesCheckerThreshold = 100; + + // Mirrors GitModule.PRIMARY_BRANCHES. + private static readonly ImmutableHashSet PrimaryBranches = + ImmutableHashSet.Create("master", "main"); + + /// Holder for the labels that should be added to the destination change message. + public class MessageInfo + { + public IReadOnlyList LabelsToAdd { get; } + + public MessageInfo(IReadOnlyList labelsToAdd) + { + LabelsToAdd = Preconditions.CheckNotNull(labelsToAdd); + } + } + + private readonly string _repoUrl; + private readonly string _fetch; + protected readonly string PushRef; + private readonly bool _partialFetch; + internal readonly bool PrimaryBranchMigrationMode; + + private readonly string? _tagName; + private readonly string? _tagMsg; + private readonly GitDestinationOptions _destinationOptions; + private readonly GitOptions _gitOptions; + private readonly GeneralOptions _generalOptions; + + private string? _resolvedPrimary; + private readonly IEnumerable _integrates; + private readonly IWriteHook _writerHook; + private readonly IChecker? _checker; + private readonly LazyResourceLoader _localRepo; + private readonly CredentialFileHandler? _credentials; + + internal GitDestination( + string repoUrl, + string fetch, + string push, + bool partialFetch, + bool primaryBranchMigrationMode, + string? tagName, + string? tagMsg, + GitDestinationOptions destinationOptions, + GitOptions gitOptions, + GeneralOptions generalOptions, + IWriteHook writerHook, + IEnumerable integrates, + IChecker? checker, + CredentialFileHandler? credentials) + { + _repoUrl = Preconditions.CheckNotNull(repoUrl); + _fetch = Preconditions.CheckNotNull(fetch); + PushRef = Preconditions.CheckNotNull(push); + _partialFetch = partialFetch; + PrimaryBranchMigrationMode = primaryBranchMigrationMode; + _tagName = tagName; + _tagMsg = tagMsg; + _destinationOptions = Preconditions.CheckNotNull(destinationOptions); + _gitOptions = Preconditions.CheckNotNull(gitOptions); + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _integrates = Preconditions.CheckNotNull(integrates); + _writerHook = Preconditions.CheckNotNull(writerHook); + _checker = checker; + _localRepo = LazyResourceLoader.Memoized( + _ => destinationOptions.LocalGitRepo(repoUrl, credentials)); + _credentials = credentials; + } + + /// + /// Throws an exception if the user.email or user.name Git configuration settings are not set. + /// + private static void VerifyUserInfoConfigured(GitRepository repo) + { + string output = repo.SimpleCommand("config", "-l").GetStdout(); + bool nameConfigured = false; + bool emailConfigured = false; + foreach (var line in output.Split('\n')) + { + if (line.StartsWith("user.name=", StringComparison.Ordinal)) + { + nameConfigured = true; + } + else if (line.StartsWith("user.email=", StringComparison.Ordinal)) + { + emailConfigured = true; + } + } + ValidationException.CheckCondition( + nameConfigured && emailConfigured, + "'user.name' and/or 'user.email' are not configured. Please run `git config --global" + + " SETTING VALUE` to set them"); + } + + public IDestination.IWriter NewWriter(WriterContext writerContext) + { + var state = new WriterState( + _localRepo, _destinationOptions.GetLocalBranch(GetPush(), writerContext.IsDryRun())); + + return new WriterImpl( + writerContext.IsDryRun(), + _repoUrl, + GetFetch(), + GetPush(), + _partialFetch, + _tagName, + _tagMsg, + _generalOptions, + _gitOptions, + _writerHook, + state, + _destinationOptions.NonFastForwardPush, + _integrates, + _destinationOptions.LastRevFirstParent, + _destinationOptions.IgnoreIntegrationErrors, + _destinationOptions.LocalRepoPath, + _destinationOptions.CommitterName, + _destinationOptions.CommitterEmail, + _destinationOptions.RebaseWhenBaseline(), + _gitOptions.VisitChangePageSize, + _gitOptions.GitTagOverwrite, + _checker, + _destinationOptions, + _credentials); + } + + /// State to be maintained between writer instances. + public class WriterState + { + internal bool AlreadyFetched; + internal bool FirstWrite = true; + internal readonly LazyResourceLoader LocalRepo; + internal readonly string LocalBranch; + + internal WriterState(LazyResourceLoader localRepo, string localBranch) + { + LocalRepo = localRepo; + LocalBranch = localBranch; + } + } + + /// A write hook allows customizing the behavior of the git.destination writer. + public interface IWriteHook + { + /// Customize the writer for a particular destination. + MessageInfo GenerateMessageInfo(TransformResult transformResult); + + /// Validate or modify the current change to be pushed. + void BeforePush( + GitRepository repo, + MessageInfo messageInfo, + bool skipPush, + IReadOnlyList integrateLabels, + IReadOnlyList originChanges) + { + } + + /// Construct the reference to push based on the pushToRefsFor reference. + string GetPushReference( + GitRepository primaryBranch, string pushToRefsFor, TransformResult transformResult); + + /// Process the server response from the push command and compute the effects. + IReadOnlyList AfterPush( + string serverResponse, + MessageInfo messageInfo, + GitRevision pushedRevision, + IReadOnlyList originChanges); + + IEndpoint GetFeedbackEndPoint(Console console) => IEndpoint.NoopEndpoint; + + ImmutableListMultimap Describe() => + ImmutableListMultimap.Empty; + } + + /// A write hook for standard git repositories. + public class DefaultWriteHook : IWriteHook + { + public MessageInfo GenerateMessageInfo(TransformResult transformResult) + { + IRevision rev = transformResult.GetCurrentRevision(); + return new MessageInfo( + transformResult.IsSetRevId() + ? new List + { + new(transformResult.GetRevIdLabel() + OriginLabelSeparator + rev.AsString()), + } + : new List()); + } + + public virtual IReadOnlyList AfterPush( + string serverResponse, + MessageInfo messageInfo, + GitRevision pushedRevision, + IReadOnlyList originChanges) => + ImmutableArray.Create( + new DestinationEffect( + DestinationEffect.EffectType.CREATED, + $"Created revision {pushedRevision.GetHash()}", + originChanges.Cast().ToList(), + new DestinationEffect.DestinationRef( + pushedRevision.GetHash(), "commit", url: null))); + + public string GetPushReference( + GitRepository repo, string pushToRefsFor, TransformResult transformResult) => + pushToRefsFor; + + public virtual IEndpoint GetFeedbackEndPoint(Console console) => IEndpoint.NoopEndpoint; + + public virtual ImmutableListMultimap Describe() => + ImmutableListMultimap.Empty; + } + + /// + /// A writer for git.*destination destinations. Not a public interface; don't use directly. + /// + public class WriterImpl : IDestination.IWriter + where TS : WriterState + { + internal readonly bool SkipPush; + private readonly string _repoUrl; + private readonly string _remoteFetch; + private readonly string _remotePush; + private readonly string? _tagNameTemplate; + private readonly string? _tagMsgTemplate; + private readonly bool _force; + private readonly bool _partialFetch; + private readonly Console _baseConsole; + private readonly GeneralOptions _generalOptions; + private readonly GitOptions _gitOptions; + private readonly IWriteHook _writeHook; + internal readonly TS State; + private readonly bool _nonFastForwardPush; + private readonly IEnumerable _integrates; + private readonly bool _lastRevFirstParent; + private readonly bool _ignoreIntegrationErrors; + private readonly string? _localRepoPath; + private readonly string _committerName; + private readonly string _committerEmail; + private readonly bool _rebase; + private readonly int _visitChangePageSize; + private readonly bool _gitTagOverwrite; + private readonly IChecker? _checker; + private readonly GitDestinationOptions _destinationOptions; + + internal WriterImpl( + bool skipPush, + string repoUrl, + string remoteFetch, + string remotePush, + bool partialFetch, + string? tagNameTemplate, + string? tagMsgTemplate, + GeneralOptions generalOptions, + GitOptions gitOptions, + IWriteHook writeHook, + TS state, + bool nonFastForwardPush, + IEnumerable integrates, + bool lastRevFirstParent, + bool ignoreIntegrationErrors, + string? localRepoPath, + string committerName, + string committerEmail, + bool rebase, + int visitChangePageSize, + bool gitTagOverwrite, + IChecker? checker, + GitDestinationOptions destinationOptions, + CredentialFileHandler? credentials) + { + SkipPush = skipPush; + _repoUrl = Preconditions.CheckNotNull(repoUrl); + _remoteFetch = Preconditions.CheckNotNull(remoteFetch); + _remotePush = Preconditions.CheckNotNull(remotePush); + _partialFetch = partialFetch; + _tagNameTemplate = tagNameTemplate; + _tagMsgTemplate = tagMsgTemplate; + _force = generalOptions.IsForced(); + _baseConsole = Preconditions.CheckNotNull(generalOptions.GetConsole()); + _generalOptions = generalOptions; + _gitOptions = Preconditions.CheckNotNull(gitOptions); + _writeHook = Preconditions.CheckNotNull(writeHook); + State = Preconditions.CheckNotNull(state); + _nonFastForwardPush = nonFastForwardPush; + _integrates = Preconditions.CheckNotNull(integrates); + _lastRevFirstParent = lastRevFirstParent; + _ignoreIntegrationErrors = ignoreIntegrationErrors; + _localRepoPath = localRepoPath; + _committerName = committerName; + _committerEmail = committerEmail; + _rebase = rebase; + _visitChangePageSize = visitChangePageSize; + _gitTagOverwrite = gitTagOverwrite; + _checker = checker; + _destinationOptions = Preconditions.CheckNotNull(destinationOptions); + } + + public void VisitChanges(GitRevision? start, IChangesVisitor visitor) + { + GitRepository repository = GetRepository(_baseConsole); + try + { + FetchIfNeeded(repository, _baseConsole); + } + catch (ValidationException e) + { + throw new CannotResolveRevisionException( + "Cannot visit changes because fetch failed. Does the destination branch exist?", + e); + } + GitRevision? startRef = GetLocalBranchRevision(repository); + if (startRef == null) + { + return; + } + ChangeReader.Builder queryChanges = + ChangeReader.Builder.ForDestination(repository, _baseConsole); + + GitVisitorUtil.VisitChanges( + start ?? startRef, + visitor, + queryChanges, + _generalOptions, + "destination", + _visitChangePageSize); + } + + private void FetchIfNeeded(GitRepository repo, Console console) + { + if (!State.AlreadyFetched) + { + GitRevision? revision = FetchFromRemote(console, repo, _repoUrl, _remoteFetch); + if (revision != null) + { + try + { + repo.Branch(State.LocalBranch).WithStartPoint(revision.GetHash()).Run(); + } + catch (RepoException e) + { + if (e.Message.Contains($"{State.LocalBranch} already exists")) + { + return; + } + throw; + } + } + State.AlreadyFetched = true; + } + } + + public DestinationStatus? GetDestinationStatus(Glob destinationFiles, string labelName) + { + GitRepository repo = GetRepository(_baseConsole); + try + { + FetchIfNeeded(repo, _baseConsole); + } + catch (AccessValidationException) + { + throw; + } + catch (ValidationException e) + { + _baseConsole.WarnFmt("Error caught when fetching from destination: {0}", e.Message); + return null; + } + GitRevision? startRef = GetLocalBranchRevision(repo); + if (startRef == null) + { + return null; + } + + var pathMatcher = destinationFiles.RelativeTo(""); + var visitor = new DestinationStatusVisitor(pathMatcher, labelName); + ChangeReader.Builder changeReader = + ChangeReader.Builder.ForDestination(repo, _baseConsole) + .SetFirstParent(_lastRevFirstParent) + .Grep("^" + labelName + OriginLabelSeparator); + try + { + GitVisitorUtil.VisitChanges( + startRef, + visitor, + changeReader, + _generalOptions, + "get_destination_status", + _visitChangePageSize); + } + catch (CannotResolveRevisionException e) + { + _baseConsole.WarnFmt("Error caught when visiting changes: {0}", e.Message); + return null; + } + return visitor.GetDestinationStatus(); + } + + public virtual IEndpoint GetFeedbackEndPoint(Console console) => + _writeHook.GetFeedbackEndPoint(console); + + private GitRevision? GetLocalBranchRevision(GitRepository gitRepository) + { + try + { + return gitRepository.ResolveReference(State.LocalBranch); + } + catch (CannotResolveRevisionException) + { + if (_force) + { + return null; + } + throw new RepoException( + $"Could not find {_remoteFetch} in {_repoUrl} and '{GeneralOptions.Force}' was" + + " not used"); + } + } + + public bool SupportsHistory() => true; + + public virtual IReadOnlyList Write( + TransformResult transformResult, Glob destinationFiles, Console console) + { + string? baseline = transformResult.GetBaseline(); + GitRepository scratchClone = GetRepository(console); + FetchIfNeeded(scratchClone, console); + + console.ProgressFmt("Git Destination: Checking out {0}", _remoteFetch); + + GitRevision? localBranchRevision = GetLocalBranchRevision(scratchClone); + UpdateLocalBranchToBaseline(scratchClone, baseline); + if (State.FirstWrite) + { + string reference = baseline ?? State.LocalBranch; + ConfigForPush(GetRepository(console), _repoUrl, _remotePush); + if (!_force && localBranchRevision == null) + { + throw new RepoException( + $"Cannot checkout '{reference}' from '{_repoUrl}'. Use" + + $" '{GeneralOptions.Force}' if the destination is a new git repo or you" + + " don't care about the destination current status"); + } + if (localBranchRevision != null) + { + scratchClone.SimpleCommand( + GetMaxRepoTimeout(), "checkout", "-f", "-q", reference); + } + else + { + // Configure the commit to go to local branch instead of main branch. + scratchClone.SimpleCommand( + "symbolic-ref", "HEAD", GetCompleteRef(State.LocalBranch)); + } + State.FirstWrite = false; + } + else + { + if (!SkipPush) + { + FetchFromRemote(console, scratchClone, _repoUrl, _remoteFetch); + } + // Checkout again in case the origin checkout changed the branch (origin = destination) + if (string.IsNullOrEmpty(scratchClone.GetCurrentBranch())) + { + scratchClone.SimpleCommand( + GetMaxRepoTimeout(), "checkout", "-q", "-f", State.LocalBranch); + } + } + var pathMatcher = destinationFiles.RelativeTo(scratchClone.GetWorkTree()!); + // Get the submodules before we stage them for deletion with add --all. + var excludedAdder = new AddExcludedFilesToIndex(scratchClone, pathMatcher); + excludedAdder.Prepare(transformResult.GetPath()); + excludedAdder.FindSubmodules(console); + + GitRepository alternate = scratchClone.WithWorkTree(transformResult.GetPath()); + + console.Progress("Git Destination: Adding all files"); + using (_generalOptions.Profiler().Start("add_files")) + { + alternate.Add().Force().All().Run(); + } + + console.Progress("Git Destination: Excluding files"); + using (_generalOptions.Profiler().Start("exclude_files")) + { + excludedAdder.Add(); + } + + console.Progress("Git Destination: Creating a local commit"); + MessageInfo messageInfo = _writeHook.GenerateMessageInfo(transformResult); + + alternate.Commit( + transformResult.GetAuthor().ToString(), + transformResult.GetTimestamp(), + AddDestinationLabels( + messageInfo, + transformResult.GetSummary().Trim().Length == 0 + ? "Internal change" + : transformResult.GetSummary())); + + MaybeCheckHeadCommit(alternate, transformResult.GetSummary(), messageInfo); + + var integrateLabels = new List(); + foreach (var integrate in _integrates) + { + IIntegrateLabel? integrateLabel = + integrate.Run( + alternate, + _repoUrl, + _generalOptions, + messageInfo, + path => !pathMatcher.Matches(Path.Combine(scratchClone.GetWorkTree()!, path)), + transformResult, + _ignoreIntegrationErrors); + + if (integrateLabel != null) + { + integrateLabels.Add(integrateLabel); + } + } + + ValidationException.CheckCondition( + transformResult.GetSummary().Trim().Length != 0, + "Change description is empty - this can be the result of scrubbing or an origin" + + " change without description."); + + // Don't leave unstaged/untracked files in the work-tree. + scratchClone.SimpleCommand("reset", "--hard"); + scratchClone.ForceClean(); + + GitRevision? afterRebaseRev = null; + if (baseline != null && _rebase) + { + var rebaseLocks = new[] + { + Path.Combine(alternate.GetGitDir(), "rebase-apply"), + Path.Combine(alternate.GetGitDir(), "rebase-merge"), + }; + foreach (var rebaseLock in rebaseLocks) + { + if (Directory.Exists(rebaseLock) || File.Exists(rebaseLock)) + { + console.Warn("Removing previous rebase failure lock: " + rebaseLock); + FileUtil.DeleteRecursively(rebaseLock); + } + } + + alternate.SimpleCommand("reset", "--hard"); + ValidationException.CheckCondition( + localBranchRevision != null, + "Unable to rebase because the local branch's revision was not resolvable."); + alternate + .RebaseCmdFor(localBranchRevision!.GetHash()) + .ErrorAdvice( + "Please consider to use flag --nogit-destination-rebase to workaround") + .Run(); + afterRebaseRev = alternate.ResolveReference("HEAD"); + if (afterRebaseRev.GetHash() == localBranchRevision.GetHash()) + { + throw new EmptyChangeException( + "Empty change after rebase. The only affected paths were already applied in" + + " main branch. This usually happens if in presubmit workflows where" + + " the used config file is more up-to-date than the origin change" + + " baseline."); + } + } + + string localBranchName = ""; + if (_localRepoPath != null) + { + if (afterRebaseRev != null) + { + localBranchName = "copybara/local"; + alternate.SimpleCommand( + GetMaxRepoTimeout(), "checkout", "-B", localBranchName, + afterRebaseRev.GetHash()); + } + scratchClone.SimpleCommand(GetMaxRepoTimeout(), "checkout", State.LocalBranch); + } + + if (transformResult.IsConfirmedInOrigin()) + { + // Diffs were shown and approved in origin. + } + else if (transformResult.IsAskForConfirmation()) + { + console.Info( + DiffUtil.Colorize( + console, scratchClone.SimpleCommand("show", "HEAD").GetStdout())); + if (!console.PromptConfirmationFmt( + "Proceed with push to {0} {1}?", _repoUrl, _remotePush)) + { + console.Warn("Migration aborted by user."); + throw new ChangeRejectedException( + "User aborted execution: did not confirm diff changes."); + } + } + + GitRevision head = scratchClone.ResolveReference("HEAD"); + IReadOnlyList originChanges = transformResult.GetChanges().GetCurrent(); + string? tagName = CreateTag(scratchClone, console, transformResult); + _writeHook.BeforePush(scratchClone, messageInfo, SkipPush, integrateLabels, originChanges); + + if (SkipPush) + { + console.InfoFmt( + "Git Destination: skipped push to remote. Check the local commits by running:" + + " GIT_DIR={0} git log {1}", + scratchClone.GetGitDir(), + localBranchName); + return ImmutableArray.Create( + new DestinationEffect( + DestinationEffect.EffectType.CREATED, + $"Dry run commit '{head}' created locally at {scratchClone.GetGitDir()}", + originChanges.Cast().ToList(), + new DestinationEffect.DestinationRef(head.GetHash(), "commit", url: null))); + } + string push = + _writeHook.GetPushReference(scratchClone, GetCompleteRef(_remotePush), transformResult); + console.Progress($"Git Destination: Pushing to {_repoUrl} {push}"); + ValidationException.CheckCondition( + !_nonFastForwardPush || _remoteFetch != _remotePush, + "non fast-forward push is only allowed when fetch != push"); + + string capturedTag = tagName!; + string capturedPush = push; + string serverResponse = + _generalOptions.RepoTask( + "push", + () => + scratchClone + .Push() + .WithRefspecs( + _repoUrl, + capturedTag != null + ? new[] + { + scratchClone.CreateRefSpec( + (_nonFastForwardPush ? "+" : "") + "HEAD:" + capturedPush), + scratchClone.CreateRefSpec( + (_gitTagOverwrite ? "+" : "") + capturedTag), + } + : new[] + { + scratchClone.CreateRefSpec( + (_nonFastForwardPush ? "+" : "") + "HEAD:" + capturedPush), + }) + .WithPushOptions(_gitOptions.GitPushOptions.ToImmutableArray()) + .Run()); + return _writeHook.AfterPush(serverResponse, messageInfo, head, originChanges); + } + + private string AddDestinationLabels(MessageInfo messageInfo, string summary) + { + ChangeMessage msg = ChangeMessage.ParseMessage(summary); + foreach (var label in messageInfo.LabelsToAdd) + { + msg = msg.WithNewOrReplacedLabel( + label.GetName(), label.GetSeparator(), label.GetValue()); + } + return msg.ToString(); + } + + private TimeSpan GetMaxRepoTimeout() => + _generalOptions.RepoTimeout > _generalOptions.CommandsTimeout + ? _generalOptions.RepoTimeout + : _generalOptions.CommandsTimeout; + + /// + /// Given a change in HEAD, if a checker is configured, it checks the affected files and the + /// commit message. + /// + private void MaybeCheckHeadCommit( + GitRepository alternate, string beforeCommitMsg, MessageInfo messageInfo) + { + if (_checker == null) + { + return; + } + var head = alternate.Log("HEAD").WithLimit(1).IncludeFiles(true).IncludeBody(true).Run(); + var commit = head[0]; + var files = commit.Files; + string target = alternate.GetWorkTree()!; + // If only a few files, create a copy so the checker doesn't check the whole tree. + if (files != null && files.Count < SmallNumFilesCheckerThreshold) + { + string dest = _generalOptions.GetDirFactory().NewTempDir("git_dest_checker"); + FileUtil.CopyFilesRecursively( + alternate.GetWorkTree()!, + dest, + FileUtil.CopySymlinkStrategy.IgnoreInvalidSymlinks, + Glob.CreateSingleFilesGlob(files)); + target = dest; + } + _checker.DoCheck(target, _baseConsole); + + // TODO(peer): DescriptionChecker processing is owned by the checks peer port. Wire it up + // once DescriptionChecker is available. + } + + private string? CreateTag( + GitRepository gitRepository, Console console, TransformResult transformResult) + { + if (_tagNameTemplate == null) + { + return null; + } + + string? tagName = null; + string? tagMsg = null; + try + { + tagName = LabelFinder.MapLabels( + transformResult.GetLabelFinder(), _tagNameTemplate); + if (_tagMsgTemplate != null) + { + tagMsg = LabelFinder.MapLabels( + transformResult.GetLabelFinder(), _tagMsgTemplate); + } + } + catch (ValidationException e) + { + console.WarnFmt("Get label failed. Error: {0}", e.Message); + } + if (tagName == null) + { + return null; + } + + try + { + if (tagMsg == null) + { + gitRepository.Tag(tagName).Force(_gitTagOverwrite).Run(); + } + else + { + gitRepository.Tag(tagName).WithAnnotatedTag(tagMsg).Force(_gitTagOverwrite).Run(); + } + return tagName; + } + catch (Exception e) when (e is RepoException or ValidationException) + { + if (e.Message.Contains($"tag '{tagName}' already exists")) + { + console.WarnFmt( + "Tag {0} exists. To overwrite it please use flag '--git-tag-overwrite'", + _tagNameTemplate); + } + else + { + console.WarnFmt( + "Create tag failed. Error: {0}. Note that we don't want to fail because of" + + " this", + e.Message); + } + return null; + } + } + + /// Get the local associated with the writer. + public GitRepository GetRepository(Console console) => State.LocalRepo.Load(console); + + private void UpdateLocalBranchToBaseline(GitRepository repo, string? baseline) + { + if (baseline != null && !repo.RefExists(baseline)) + { + throw new RepoException( + "Cannot find baseline '" + baseline + + (GetLocalBranchRevision(repo) != null + ? "' from fetch reference '" + _remoteFetch + "'" + : "' and fetch reference '" + _remoteFetch + "' itself") + + " in " + _repoUrl + "."); + } + if (baseline != null) + { + repo.SimpleCommand("update-ref", State.LocalBranch, baseline); + } + } + + private GitRevision? FetchFromRemote( + Console console, GitRepository repo, string repoUrl, string fetch) + { + string completeFetchRef = GetCompleteRef(fetch); + using (_generalOptions.Profiler().Start("destination_fetch")) + { + console.Progress("Git Destination: Fetching: " + repoUrl + " " + completeFetchRef); + try + { + return repo.FetchSingleRef( + repoUrl, completeFetchRef, _partialFetch, _destinationOptions.GetFetchDepth()); + } + catch (CannotResolveRevisionException) + { + string warning = + $"Git Destination: '{completeFetchRef}' doesn't exist in '{repoUrl}'"; + ValidationException.CheckCondition( + _force, + "{0}. Use {1} flag if you want to push anyway", + warning, + GeneralOptions.Force); + console.Warn(warning); + } + } + return null; + } + + private static string GetCompleteRef(string fetch) => + fetch.StartsWith("refs/", StringComparison.Ordinal) ? fetch : "refs/heads/" + fetch; + + private void ConfigForPush(GitRepository repo, string repoUrl, string push) + { + if (_localRepoPath != null) + { + repo.SimpleCommand("config", "remote.copybara_remote.url", repoUrl); + repo.SimpleCommand( + "config", "remote.copybara_remote.push", State.LocalBranch + ":" + push); + repo.SimpleCommand( + "config", "branch." + State.LocalBranch + ".remote", "copybara_remote"); + } + if (!string.IsNullOrEmpty(_committerName)) + { + repo.SimpleCommand("config", "user.name", _committerName); + } + if (!string.IsNullOrEmpty(_committerEmail)) + { + repo.SimpleCommand("config", "user.email", _committerEmail); + } + VerifyUserInfoConfigured(repo); + } + + public DestinationReader GetDestinationReader( + Console console, Origin.Baseline? baseline, string workdir) => + GetDestinationReader(console, baseline?.GetBaseline(), workdir); + + public DestinationReader GetDestinationReader( + Console console, string? baseline, string workdir) + { + GitRepository repo = GetRepository(console); + FetchIfNeeded(repo, console); + GitRevision? rev; + if (baseline != null) + { + rev = repo.ResolveReference(baseline); + } + else + { + rev = GetLocalBranchRevision(repo); + } + // In case of --force, the destination might be empty and have no revisions. Do not fail. + if (rev == null) + { + console.Info( + "Destination reader requested, but destination is empty. Using noop reader"); + return DestinationReader.NoopDestinationReader; + } + return new GitDestinationReader(repo, rev, workdir); + } + } + + internal string GetFetch() + { + if (PrimaryBranchMigrationMode && PrimaryBranches.Contains(_fetch)) + { + string? resolved = GetResolvedPrimary(); + if (resolved != null) + { + return resolved; + } + } + return _fetch; + } + + internal string GetPush() + { + if (PrimaryBranchMigrationMode && PrimaryBranches.Contains(PushRef)) + { + string? resolved = GetResolvedPrimary(); + if (resolved != null) + { + return resolved; + } + } + return PushRef; + } + + protected string? GetResolvedPrimary() + { + if (_resolvedPrimary == null) + { + try + { + _resolvedPrimary = + GetLocalRepo().Load(_generalOptions.GetConsole()).GetPrimaryBranch(_repoUrl); + } + catch (RepoException) + { + return null; + } + } + return _resolvedPrimary; + } + + public IEnumerable GetIntegrates() => _integrates; + + public string GetLabelNameWhenOrigin() => GitRepository.GitOriginRevId; + + public override string ToString() => + $"GitDestination{{repoUrl={_repoUrl}, fetch={_fetch}, push={PushRef}," + + $" partialFetch={_partialFetch}, primaryBranchMigrationMode={PrimaryBranchMigrationMode}}}"; + + public IWriteHook GetWriterHook() => _writerHook; + + /// Not a public API. It is subject to change. + public LazyResourceLoader GetLocalRepo() => _localRepo; + + public string GetType() => "git.destination"; + + public ImmutableListMultimap Describe(Glob? destinationFiles) + { + var builder = ImmutableListMultimap.CreateBuilder(); + builder.Put("type", GetType()); + builder.Put("url", _repoUrl); + builder.Put("fetch", _fetch); + builder.Put("push", PushRef); + builder.Put("primaryBranchMigrationMode", PrimaryBranchMigrationMode.ToString()); + builder.PutAll(_writerHook.Describe()); + if (destinationFiles != null + && !destinationFiles.Roots().IsEmpty + && !destinationFiles.Roots().Contains("")) + { + builder.PutAll("root", destinationFiles.Roots()); + } + if (_partialFetch) + { + builder.Put("partialFetch", _partialFetch.ToString()); + } + if (_tagName != null) + { + builder.Put("tagName", _tagName); + } + if (_tagMsg != null) + { + builder.Put("tagMsg", _tagMsg); + } + if (_checker != null) + { + builder.Put("checker", _checker.GetType().FullName ?? _checker.GetType().Name); + } + foreach (var integrate in _integrates) + { + builder.Put("integrate", $"{integrate.GetLabel()}:{integrate.GetStrategy()}"); + } + + return builder.Build(); + } + + public IReadOnlyList> DescribeCredentials() + { + if (_credentials == null) + { + return ImmutableArray>.Empty; + } + return GitDescribeCredentials.Convert(_credentials.DescribeCredentials()); + } +} diff --git a/src/Copybara.Core/Git/GitDestinationOptions.cs b/src/Copybara.Core/Git/GitDestinationOptions.cs new file mode 100644 index 000000000..19be06a76 --- /dev/null +++ b/src/Copybara.Core/Git/GitDestinationOptions.cs @@ -0,0 +1,173 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Authoring; +using Copybara.Common; +using Copybara.Exceptions; + +namespace Copybara.Git; + +/// +/// Arguments for , , and other Git components. +/// Port of com.google.copybara.git.GitDestinationOptions. +/// +public sealed class GitDestinationOptions : IOption +{ + private readonly GeneralOptions _generalOptions; + private readonly GitOptions _gitOptions; + + [Flag( + "--git-committer-name", + "If set, overrides the committer name for the generated commits in git destination.")] + public string CommitterName { get; set; } = ""; + + [Flag( + "--git-committer-email", + "If set, overrides the committer e-mail for the generated commits in git destination.")] + public string CommitterEmail { get; set; } = ""; + + [Flag( + "--git-skip-checker", + "If true and git.destination has a configured checker, it will not be used in the" + + " migration.", + Arity = 1)] + public bool SkipGitChecker { get; set; } + + public GitDestinationOptions(GeneralOptions generalOptions, GitOptions gitOptions) + { + _generalOptions = Preconditions.CheckNotNull(generalOptions); + _gitOptions = Preconditions.CheckNotNull(gitOptions); + } + + internal Author GetCommitter() => new(CommitterName, CommitterEmail); + + [Flag("--git-destination-url", "If set, overrides the git destination URL.")] + internal string? Url { get; set; } + + [Flag( + "--git-destination-fetch", + "If set, overrides the git destination fetch reference.")] + public string? Fetch { get; set; } + + [Flag( + "--git-destination-push", + "If set, overrides the git destination push reference.")] + public string? Push { get; set; } + + [Flag( + "--git-destination-path", + "If set, the tool will use this directory for the local repository. Note that if the" + + " directory exists it needs to be a git repository. Copybara will revert any" + + " staged/unstaged changes. For example, you can override destination url with a local" + + " non-bare repo (or existing empty folder) with this flag.")] + public string? LocalRepoPath { get; set; } + + [Flag( + "--git-destination-last-rev-first-parent", + "Use git --first-parent flag when looking for last-rev in previous commits")] + internal bool LastRevFirstParent { get; set; } + + [Flag( + "--git-destination-non-fast-forward", + "Allow non-fast-forward pushes to the destination. We only allow this when used with" + + " different push != fetch references.")] + internal bool NonFastForwardPush { get; set; } + + [Flag( + "--git-destination-ignore-integration-errors", + "If an integration error occurs, ignore it and continue without the integrate")] + internal bool IgnoreIntegrationErrors { get; set; } + + [Flag( + "--nogit-destination-rebase", + "Don't rebase the change automatically for workflows CHANGE_REQUEST mode")] + public bool NoRebase { get; set; } + + [Flag( + "--git-destination-fetch-depth", + "Use a shallow clone of the specified depth for git.destination")] + internal int? FetchDepth { get; set; } + + public int? GetFetchDepth() => FetchDepth; + + internal bool RebaseWhenBaseline() => !NoRebase; + + /// + /// Returns a non-bare repo. Either because it uses a custom worktree or because it is a user + /// non-bare repo. + /// + internal GitRepository LocalGitRepo(string url, CredentialFileHandler? creds) + { + GitRepository repo = GetLocalGitRepository(url); + if (creds != null) + { + try + { + creds.Install(repo, _gitOptions.GetConfigCredsFile(_generalOptions)); + } + catch (IOException e) + { + throw new RepoException("Unable to store git credentials", e); + } + } + return repo; + } + + private GitRepository GetLocalGitRepository(string url) + { + try + { + if (string.IsNullOrEmpty(LocalRepoPath)) + { + return _gitOptions.CachedBareRepoForUrl(url) + .WithWorkTree(_generalOptions.GetDirFactory().NewTempDir("git_dest")); + } + string path = LocalRepoPath; + + if (!Directory.Exists(path) || (Directory.Exists(path) && IsGitRepoOrEmptyDir(path))) + { + Directory.CreateDirectory(path); + return _gitOptions.InitRepo( + GitRepository.NewRepo( + _generalOptions.IsVerbose(), + path, + _gitOptions.GetGitEnvironment(_generalOptions.GetEnvironment()), + _generalOptions.RepoTimeout, + _gitOptions.GitNoVerify, + _gitOptions.GetPushOptionsValidator())); + } + throw new RepoException(path + " is not empty and is not a git repository"); + } + catch (IOException e) + { + throw new RepoException("Cannot create local repository", e); + } + } + + private static bool IsGitRepoOrEmptyDir(string path) => + Directory.Exists(Path.Combine(path, ".git")) + || (!Directory.EnumerateFileSystemEntries(path).Any()); + + /// Used internally to be able to traverse the local repo once a migration finishes. + public string? CustomLocalBranch { get; set; } + + /// Returns the local branch that will be used for working on the change before pushing. + internal string GetLocalBranch(string resolvedPush, bool dryRun) => + LocalRepoPath != null + ? resolvedPush // This is nicer for the user + : CustomLocalBranch + ?? "copybara/resolvedPush-" + Guid.NewGuid() + (dryRun ? "-dryrun" : ""); +} diff --git a/src/Copybara.Core/Git/GitDestinationReader.cs b/src/Copybara.Core/Git/GitDestinationReader.cs new file mode 100644 index 000000000..a8c560141 --- /dev/null +++ b/src/Copybara.Core/Git/GitDestinationReader.cs @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Copybara.Common; +using Copybara.Config; +using Copybara.Exceptions; +using Copybara.Util; +using Starlark.Annot; + +namespace Copybara.Git; + +/// +/// A DestinationReader for reading files from a GitDestination. Port of +/// com.google.copybara.git.GitDestinationReader. +/// +[StarlarkBuiltin( + "git_destination_reader", + Doc = "Handle to read from a git destination", + Documented = false)] +public class GitDestinationReader : DestinationReader +{ + private readonly GitRepository _repository; + private readonly string _workDir; + private readonly GitRevision _baseline; + + public GitDestinationReader(GitRepository repository, GitRevision baseline, string workDir) + { + _repository = Preconditions.CheckNotNull(repository); + _baseline = Preconditions.CheckNotNull(baseline); + _workDir = Preconditions.CheckNotNull(workDir); + } + + public override string ReadFile(string path) => + _repository.ReadFile(_baseline.GetHash(), path); + + public override void CopyDestinationFiles(object glob, object path) + { + var checkoutPath = SkylarkUtil.ConvertFromNoneable(path, null); + Glob resolvedGlob = Glob.WrapGlob(glob, null)!; + if (checkoutPath == null) + { + CopyDestinationFilesToDirectory(resolvedGlob, _workDir); + } + else + { + CopyDestinationFilesToDirectory(resolvedGlob, checkoutPath.FullPath()); + } + } + + public override void CopyDestinationFilesToDirectory(Glob glob, string directory) + { + var treeElements = _repository.LsTree(_baseline, null, true, true); + var pathMatcher = glob.RelativeTo(directory); + foreach (var file in treeElements) + { + string path = Path.Combine(directory, file.Path); + if (pathMatcher.Matches(path)) + { + try + { + string? parent = Path.GetDirectoryName(path); + if (parent != null) + { + Directory.CreateDirectory(parent); + } + } + catch (IOException e) + { + throw new RepoException( + $"Cannot create parent directory for {path}", e); + } + } + } + _repository.Checkout(glob, directory, _baseline); + } + + public override bool Exists(string path) + { + try + { + return _repository.ReadFile(_baseline.GetHash(), path) != null; + } + catch (RepoException) + { + return false; + } + } + + public override string? LastModified(string path) => + _repository.LastModified(_baseline.GetHash(), path); +} diff --git a/src/Copybara.Core/Git/GitEnvironment.cs b/src/Copybara.Core/Git/GitEnvironment.cs new file mode 100644 index 000000000..0ee245dd6 --- /dev/null +++ b/src/Copybara.Core/Git/GitEnvironment.cs @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using Copybara.Common; + +namespace Copybara.Git; + +/// +/// Environment for running git commands. Port of +/// com.google.copybara.git.GitEnvironment. +/// +public sealed class GitEnvironment +{ + private readonly ImmutableDictionary _environment; + private readonly bool _noGitPrompt; + + public GitEnvironment(IReadOnlyDictionary environment) + : this(environment, noGitPrompt: false) + { + } + + public GitEnvironment(IReadOnlyDictionary environment, bool noGitPrompt) + { + _environment = Preconditions.CheckNotNull(environment).ToImmutableDictionary(); + _noGitPrompt = noGitPrompt; + } + + /// + /// Returns the environment to pass to git subprocesses. Explicitly forces the output language to + /// english so parsing of git's output succeeds independently of the user's default locale. + /// + public ImmutableDictionary GetEnvironment() + { + var env = new Dictionary(_environment); + + // Explicitly set output language to english so parsing of git's output succeeds + // independently of users default locale. + env["LANG"] = "en_US.UTF-8"; + + if (_noGitPrompt) + { + env["GIT_TERMINAL_PROMPT"] = "0"; + } + + return env.ToImmutableDictionary(); + } + + /// + /// Returns a copy of this environment, with the given vars added. If a key is already present, + /// its value is overwritten. + /// + public GitEnvironment WithVars(IReadOnlyDictionary vars) + { + var allVars = new Dictionary(_environment); + foreach (var kvp in vars) + { + allVars[kvp.Key] = kvp.Value; + } + return new GitEnvironment(allVars, _noGitPrompt); + } + + /// + /// Returns a copy of this environment, setting explicitly to prevent Git from asking for + /// username/password and fail if the credentials cannot be resolved. + /// + public GitEnvironment WithNoGitPrompt() => new(_environment, noGitPrompt: true); + + /// + /// Returns a String representing the git binary to be executed. + /// + /// The env var GIT_EXEC_PATH determines where Git looks for its sub-programs, but + /// also the regular git binaries (git, git-upload-pack, etc) are duplicated in + /// GIT_EXEC_PATH. + /// + /// If the env var is not set, then we will execute "git", that it will be resolved in the + /// path as usual. + /// + public string ResolveGitBinary() + { + if (_environment.TryGetValue("GIT_EXEC_PATH", out var execPath)) + { + return Path.Combine(execPath, "git"); + } + return "git"; + } +} diff --git a/src/Copybara.Core/Git/GitHub/Api/AbstractGitHubApiTransport.cs b/src/Copybara.Core/Git/GitHub/Api/AbstractGitHubApiTransport.cs new file mode 100644 index 000000000..31b8fc6c3 --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/AbstractGitHubApiTransport.cs @@ -0,0 +1,311 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Copybara.Common; +using Copybara.Exceptions; +using Copybara.Git; +using ConsoleT = Copybara.Util.Console.Console; + +namespace Copybara.Git.GitHub.Api; + +/// +/// Base implementation of that uses +/// and System.Text.Json for the requests. +/// +/// +/// Port of com.google.copybara.git.github.api.AbstractGitHubApiTransport (google-http-client +/// based). Credentials are resolved through the git credential helper, exactly as upstream: a token +/// for the API host is preferred, falling back to the web host. +/// +public abstract class AbstractGitHubApiTransport : IGitHubApiTransport +{ + private const string GitHubDotComApiUrl = "https://api.github.com"; + private const string GitHubDotComWebUrl = "https://github.com"; + + protected readonly string ApiUrl; + protected readonly string WebUrl; + protected readonly GitRepository Repo; + protected readonly HttpClient HttpClient; + protected readonly string StorePath; + protected readonly ConsoleT Console; + protected readonly bool BearerAuth; + + protected AbstractGitHubApiTransport( + GitRepository repo, + HttpClient httpClient, + string storePath, + bool bearerAuth, + ConsoleT console, + string webUrl) + { + Repo = Preconditions.CheckNotNull(repo); + HttpClient = Preconditions.CheckNotNull(httpClient); + StorePath = storePath; + Console = Preconditions.CheckNotNull(console); + BearerAuth = bearerAuth; + WebUrl = BuildWebUrl(Preconditions.CheckNotNull(webUrl)); + ApiUrl = DetermineApiUrl(WebUrl); + } + + /// Sends the request, allowing subclasses to intercept it (e.g. for tests). + protected abstract Task ExecuteRequestAsync(HttpRequestMessage request); + + public async Task GetAsync( + string path, + ImmutableListMultimap headers, + string requestDescription) + { + GitCredential.UserPassword? credentials = GetCredentialsIfPresent(); + Uri url = GetFullEndpointUrl(path); + try + { + Console.VerboseFmt("Executing {0}", requestDescription); + using var request = new HttpRequestMessage(HttpMethod.Get, url); + ApplyHeaders(request, credentials, headers); + using HttpResponseMessage response = await ExecuteRequestAsync(request).ConfigureAwait(false); + await ThrowIfError(response, "GET", path, null).ConfigureAwait(false); + return await ParseResponseAsync(response).ConfigureAwait(false); + } + catch (HttpRequestException e) + { + throw new RepoException("Error running GitHub API operation " + path, e); + } + } + + public Task GetAsync(string path, string requestDescription) => + GetAsync(path, ImmutableListMultimap.Empty, requestDescription); + + public async Task PostAsync(string path, object request, string requestType) + { + GitCredential.UserPassword credentials = GetCredentials(); + Uri url = GetFullEndpointUrl(path); + string requestJson = JsonSerializer.Serialize(request, request.GetType(), GitHubApiJson.Options); + try + { + Console.VerboseFmt("Executing {0}", requestType); + using var httpRequest = new HttpRequestMessage(HttpMethod.Post, url) + { + Content = new StringContent(requestJson, Encoding.UTF8, "application/json"), + }; + ApplyHeaders(httpRequest, credentials, ImmutableListMultimap.Empty); + using HttpResponseMessage response = + await ExecuteRequestAsync(httpRequest).ConfigureAwait(false); + await ThrowIfError(response, "POST", path, requestJson).ConfigureAwait(false); + return await ParseResponseAsync(response).ConfigureAwait(false); + } + catch (HttpRequestException e) + { + throw new RepoException("Error running GitHub API operation " + path, e); + } + } + + public abstract Task DeleteAsync(string path, string requestType); + + private async Task ParseResponseAsync(HttpResponseMessage response) + { + T? responseObj = await ParseHttpResponseAsync(response).ConfigureAwait(false); + if (responseObj is IPaginatedPayload paginatedPayload) + { + return (T)paginatedPayload.AnnotatePayload(ApiUrl, MaybeGetLinkHeader(response)); + } + + return responseObj; + } + + private static async Task ParseHttpResponseAsync(HttpResponseMessage response) + { + if (response.StatusCode == HttpStatusCode.NoContent) + { + return default; + } + + byte[] bytes = await response.Content.ReadAsByteArrayAsync().ConfigureAwait(false); + if (bytes.Length == 0) + { + return default; + } + + try + { + return JsonSerializer.Deserialize(bytes, GitHubApiJson.Options); + } + catch (Exception e) when (e is JsonException or NotSupportedException) + { + throw new RepoException( + $"Cannot parse content as type {typeof(T)}.\nContent: {Encoding.UTF8.GetString(bytes)}\n", + e); + } + } + + /// Throws a if the response is not a success. + protected async Task ThrowIfError( + HttpResponseMessage response, string method, string path, string? request) + { + if (response.IsSuccessStatusCode) + { + return; + } + + string? content = response.Content == null + ? null + : await response.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new GitHubApiException( + (int)response.StatusCode, + ParseErrorOrIgnore(content), + method, + path, + request, + content); + } + + protected Uri GetFullEndpointUrl(string path) + { + string maybePrefix = path.StartsWith("/", StringComparison.Ordinal) ? "" : "/"; + return new Uri(ApiUrl + maybePrefix + path); + } + + protected static ClientError? ParseErrorOrIgnore(string? content) + { + if (content == null) + { + return null; + } + + try + { + return JsonSerializer.Deserialize(content, GitHubApiJson.Options); + } + catch (JsonException) + { + return new ClientError(); + } + } + + protected static string? MaybeGetLinkHeader(HttpResponseMessage response) + { + if (response.Headers.TryGetValues("Link", out IEnumerable? link)) + { + return link.FirstOrDefault(); + } + + return null; + } + + protected void ApplyHeaders( + HttpRequestMessage request, + GitCredential.UserPassword? userPassword, + ImmutableListMultimap headers) + { + if (userPassword != null) + { + if (BearerAuth) + { + request.Headers.Authorization = + new AuthenticationHeaderValue("Bearer", userPassword.GetPasswordBeCareful()); + } + else + { + string basic = Convert.ToBase64String( + Encoding.ASCII.GetBytes( + userPassword.GetUsername() + ":" + userPassword.GetPasswordBeCareful())); + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", basic); + } + } + + foreach (KeyValuePair header in headers) + { + request.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + } + + /// Credentials for API should be optional for any read operation (GET). + protected GitCredential.UserPassword? GetCredentialsIfPresent() + { + try + { + return GetCredentials(); + } + catch (ValidationException) + { + string msg = + $"GitHub credentials not found in {StorePath}. Assuming the repository is public."; + Console.Info(msg); + return null; + } + } + + /// + /// Gets the credentials from git credential helper. First tries the apiUrl host, then falls back + /// to the webUrl host. + /// + /// + /// + protected GitCredential.UserPassword GetCredentials() + { + try + { + return Repo.CredentialFill(ApiUrl); + } + catch (ValidationException) + { + try + { + return Repo.CredentialFill(WebUrl); + } + catch (ValidationException e1) + { + throw new ValidationException( + $"Cannot get credentials for host {WebUrl} or {ApiUrl} from credentials helper." + + " Make sure either your credential helper has the username and password/token" + + $" or if you don't use one, that file '{StorePath}' contains one of the two" + + " lines: \nEither:\n" + + $"https://USERNAME:TOKEN@{RemoveHttpsPrefix(ApiUrl)}\n" + + "or:\n" + + $"https://USERNAME:TOKEN@{RemoveHttpsPrefix(WebUrl)}\n" + + "\n" + + "Note that spaces or other special characters need to be escaped. For example" + + " ' ' should be %20 and '@' should be %40 (For example when using the email" + + " as username)", + e1); + } + } + } + + private static string RemoveHttpsPrefix(string url) => url.Replace("https://", ""); + + private static string BuildWebUrl(string hostName) => "https://" + hostName; + + private static string DetermineApiUrl(string hostName) + { + // Github.com has a unique API URL. + // GitHub Enterprise instances have a specific format of API URL. + if (hostName == GitHubDotComWebUrl) + { + return GitHubDotComApiUrl; + } + + return hostName + "/api/v3"; + } + + public string GetApiUrl() => ApiUrl; + + public string GetWebUrl() => WebUrl; +} diff --git a/src/Copybara.Core/Git/GitHub/Api/AddAssignees.cs b/src/Copybara.Core/Git/GitHub/Api/AddAssignees.cs new file mode 100644 index 000000000..40bf3b722 --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/AddAssignees.cs @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace Copybara.Git.GitHub.Api; + +/// +/// An object that represents the list of github users to assign to a pull request or issue. +/// +public class AddAssignees +{ + [JsonPropertyName("assignees")] + public List? Assignees { get; set; } + + public List? GetAssignees() => Assignees; + + public void SetAssignees(List assignees) => Assignees = assignees; + + public AddAssignees(List assignees) + { + Assignees = assignees; + } + + public AddAssignees() + { + } +} diff --git a/src/Copybara.Core/Git/GitHub/Api/AddLabels.cs b/src/Copybara.Core/Git/GitHub/Api/AddLabels.cs new file mode 100644 index 000000000..855eeb12c --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/AddLabels.cs @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.Json.Serialization; +using Copybara.Common; + +namespace Copybara.Git.GitHub.Api; + +/// +/// Request type for adding a label to an issue. +/// https://developer.github.com/v3/issues/labels/#add-labels-to-an-issue +/// +public class AddLabels +{ + [JsonPropertyName("labels")] + public List? Labels { get; set; } + + public AddLabels(IReadOnlyList labels) + { + Preconditions.CheckNotNull(labels); + Labels = new List(labels); + } + + public AddLabels() + { + } + + public IReadOnlyList GetLabels() => + Labels == null ? ImmutableArray.Empty : Labels.ToImmutableArray(); +} diff --git a/src/Copybara.Core/Git/GitHub/Api/AuthorAssociation.cs b/src/Copybara.Core/Git/GitHub/Api/AuthorAssociation.cs new file mode 100644 index 000000000..9ad919c6f --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/AuthorAssociation.cs @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Copybara.Git.GitHub.Api; + +/// +/// GitHub type of author for pull request, reviews, etc. +/// +/// See https://developer.github.com/v4/reference/enum/commentauthorassociation/ +/// +public enum AuthorAssociation +{ + CONTRIBUTOR, + COLLABORATOR, + MEMBER, + OWNER, + NONE, + FIRST_TIMER, + FIRST_TIME_CONTRIBUTOR, +} diff --git a/src/Copybara.Core/Git/GitHub/Api/CheckRun.cs b/src/Copybara.Core/Git/GitHub/Api/CheckRun.cs new file mode 100644 index 000000000..74501de2b --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/CheckRun.cs @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GitHub.Api; + +/// +/// Represents a GitHub App's checkRun detail. +/// https://developer.github.com/v3/checks/runs/#create-a-check-run +/// https://developer.github.com/v3/checks/runs/#response +/// +[StarlarkBuiltin( + "github_check_run_obj", + Doc = + "Detail about a check run as defined in " + + "https://developer.github.com/v3/checks/runs/#create-a-check-run")] +public class CheckRun : IStarlarkValue +{ + [JsonPropertyName("details_url")] + public string? DetailUrl { get; set; } + + [JsonPropertyName("status")] + public CheckRunStatus Status { get; set; } + + [JsonPropertyName("conclusion")] + public CheckRunConclusion? Conclusion { get; set; } + + [JsonPropertyName("head_sha")] + public string? Sha { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("app")] + public GitHubApp? App { get; set; } + + [JsonPropertyName("output")] + public Output? Output { get; set; } + + [JsonPropertyName("pull_requests")] + public List? PullRequests { get; set; } + + [StarlarkMethod( + "detail_url", + Doc = "The URL of the integrator's site that has the full details of the check.", + StructField = true, + AllowReturnNones = true)] + public string? GetDetailUrl() => DetailUrl; + + [StarlarkMethod( + "status", + Doc = "The current status of the check run. Can be one of queued, in_progress, or completed.", + StructField = true)] + public string GetStatus() => Status.ToString().ToLowerInvariant(); + + [StarlarkMethod( + "conclusion", + Doc = "The final conclusion of the check. Can be one of success, failure, neutral, " + + "cancelled, timed_out, or action_required.", + StructField = true, + AllowReturnNones = true)] + public string? GetConclusion() => Conclusion?.ToString().ToLowerInvariant(); + + [StarlarkMethod("sha", Doc = "The SHA-1 the check run is based on", StructField = true)] + public string? GetSha() => Sha; + + [StarlarkMethod("name", Doc = "The name of the check", StructField = true)] + public string? GetName() => Name; + + [StarlarkMethod("app", Doc = "The detail of a GitHub App, such as id, slug, and name", StructField = true)] + public GitHubApp? GetApp() => App; + + [StarlarkMethod( + "output", + Doc = "The description of a GitHub App's run, including title, summary, text.", + StructField = true)] + public Output? GetOutput() => Output; + + [StarlarkMethod( + "pulls", + Doc = "Pull requests associated with this check_run ('number' only)", + StructField = true)] + public IReadOnlyList GetPullRequests() => + PullRequests == null + ? ImmutableArray.Empty + : PullRequests.ToImmutableArray(); + + public override string ToString() => + $"CheckRun{{details_url={DetailUrl}, status={Status}, conclusion={Conclusion}, sha={Sha}," + + $" name={Name}, app={App}, output={Output}, pulls={PullRequests}}}"; + + /// PR submessage in check_run. + public class CheckRunPullRequest : IStarlarkValue + { + [JsonPropertyName("number")] + public int Number { get; set; } + + [StarlarkMethod("number", Doc = "Number of a PR liked to the check_run", StructField = true)] + public int GetNumber() => Number; + + public override string ToString() => $"PullRequest{{number={Number}}}"; + } +} + +/// Status of a check run. +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum CheckRunStatus +{ + [JsonStringEnumMemberName("queued")] + QUEUED, + + [JsonStringEnumMemberName("in_progress")] + IN_PROGRESS, + + [JsonStringEnumMemberName("completed")] + COMPLETED, + + [JsonStringEnumMemberName("pending")] + PENDING, +} + +/// Conclusion of a check run status. +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum CheckRunConclusion +{ + NONE, + + [JsonStringEnumMemberName("success")] + SUCCESS, + + [JsonStringEnumMemberName("failure")] + FAILURE, + + [JsonStringEnumMemberName("neutral")] + NEUTRAL, + + [JsonStringEnumMemberName("timed_out")] + TIMEDOUT, + + [JsonStringEnumMemberName("cancelled")] + CANCELLED, + + [JsonStringEnumMemberName("action_required")] + ACTIONREQUIRED, + + [JsonStringEnumMemberName("skipped")] + SKIPPED, + + [JsonStringEnumMemberName("stale")] + STALE, + + [JsonStringEnumMemberName("startup_failure")] + STARTUPFAILURE, +} + +/// Helpers for . +public static class CheckRunConclusions +{ + private static readonly IReadOnlyDictionary ApiValues = + new Dictionary + { + [CheckRunConclusion.NONE] = "none", + [CheckRunConclusion.SUCCESS] = "success", + [CheckRunConclusion.FAILURE] = "failure", + [CheckRunConclusion.NEUTRAL] = "neutral", + [CheckRunConclusion.TIMEDOUT] = "timed_out", + [CheckRunConclusion.CANCELLED] = "cancelled", + [CheckRunConclusion.ACTIONREQUIRED] = "action_required", + [CheckRunConclusion.SKIPPED] = "skipped", + [CheckRunConclusion.STALE] = "stale", + [CheckRunConclusion.STARTUPFAILURE] = "startup_failure", + }; + + public static string GetApiVal(this CheckRunConclusion conclusion) => ApiValues[conclusion]; + + /// + /// Given a String value of Conclusion as returned by GitHub API, returns the equivalent enum + /// value or null if not found. + /// + public static CheckRunConclusion? FromValue(string val) + { + foreach (var kv in ApiValues) + { + if (kv.Value == val) + { + return kv.Key; + } + } + + return null; + } +} diff --git a/src/Copybara.Core/Git/GitHub/Api/CheckRuns.cs b/src/Copybara.Core/Git/GitHub/Api/CheckRuns.cs new file mode 100644 index 000000000..4577ac999 --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/CheckRuns.cs @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GitHub.Api; + +/// +/// Represents the response of list check runs for a specific ref. +/// https://developer.github.com/v3/checks/runs/#list-check-runs-for-a-specific-ref +/// +[StarlarkBuiltin( + "github_check_runs_obj", + Doc = + "List check runs for a specific ref " + + "https://developer.github.com/v3/checks/runs/#list-check-runs-for-a-specific-ref")] +public class CheckRuns : IStarlarkValue, IPaginatedPayload +{ + [JsonPropertyName("total_count")] + public int TotalCount { get; set; } + + [JsonPropertyName("check_runs")] + public PaginatedList CheckRunsList { get; set; } = new(); + + public CheckRuns() + { + } + + private CheckRuns(int totalCount, PaginatedList checkRuns) + { + CheckRunsList = checkRuns; + TotalCount = totalCount; + } + + [StarlarkMethod("total_count", Doc = "The total count of check runs.", StructField = true)] + public int GetTotalCount() => TotalCount; + + [StarlarkMethod("check_runs", Doc = "The list of the detail for each check run.", StructField = true)] + public IReadOnlyList GetCheckRuns() => CheckRunsList; + + public override string ToString() => + $"CheckRuns{{total_count={TotalCount}, check_runs={CheckRunsList}}}"; + + public PaginatedList GetPayload() => CheckRunsList; + + public IPaginatedPayload AnnotatePayload(string apiPrefix, string? linkHeader) => + new CheckRuns(TotalCount, CheckRunsList.WithPaginationInfo(apiPrefix, linkHeader)); +} diff --git a/src/Copybara.Core/Git/GitHub/Api/CheckSuite.cs b/src/Copybara.Core/Git/GitHub/Api/CheckSuite.cs new file mode 100644 index 000000000..91a74e486 --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/CheckSuite.cs @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GitHub.Api; + +/// +/// Represents a GitHub App's checkRun detail. +/// https://docs.github.com/en/rest/checks/suites?apiVersion=2022-11-28#list-check-suites-for-a-git-reference +/// +[StarlarkBuiltin( + "github_check_suite_obj", + Doc = + "Detail about a check run as defined in " + + "https://developer.github.com/v3/checks/runs/#create-a-check-run")] +public class CheckSuite : IStarlarkValue +{ + [JsonPropertyName("status")] + public CheckRunStatus Status { get; set; } + + [JsonPropertyName("conclusion")] + public CheckRunConclusion? Conclusion { get; set; } + + [JsonPropertyName("head_sha")] + public string? Sha { get; set; } + + [JsonPropertyName("id")] + public long Id { get; set; } + + [JsonPropertyName("app")] + public GitHubApp? App { get; set; } + + [StarlarkMethod("id", Doc = "Check suite identifier", StructField = true)] + public StarlarkInt GetId() => StarlarkInt.Of(Id); + + [StarlarkMethod( + "status", + Doc = "The current status of the check run. Can be one of queued, in_progress, pending," + + " or completed.", + StructField = true)] + public string GetStatus() => Status.ToString().ToLowerInvariant(); + + [StarlarkMethod( + "conclusion", + Doc = "The final conclusion of the check. Can be one of success, failure, neutral, " + + "cancelled, timed_out, or action_required.", + StructField = true, + AllowReturnNones = true)] + public string? GetConclusion() => Conclusion?.ToString().ToLowerInvariant(); + + [StarlarkMethod("sha", Doc = "The SHA-1 the check run is based on", StructField = true)] + public string? GetSha() => Sha; + + [StarlarkMethod("app", Doc = "The detail of a GitHub App, such as id, slug, and name", StructField = true)] + public GitHubApp? GetApp() => App; + + public override string ToString() => + $"CheckSuite{{id={Id}, status={Status}, conclusion={Conclusion}, sha={Sha}, app={App}}}"; +} diff --git a/src/Copybara.Core/Git/GitHub/Api/CheckSuites.cs b/src/Copybara.Core/Git/GitHub/Api/CheckSuites.cs new file mode 100644 index 000000000..2143ec3fc --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/CheckSuites.cs @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GitHub.Api; + +/// +/// Represents a GitHub App's checkSuites response detail. +/// https://docs.github.com/en/rest/checks/suites?apiVersion=2022-11-28#list-check-suites-for-a-git-reference +/// +[StarlarkBuiltin( + "github_check_suites_response_obj", + Doc = + "Detail about a check run as defined in " + + "https://docs.github.com/en/rest/checks/suites?apiVersion=2022-11-28#list-check-suites-for-a-git-reference")] +public class CheckSuites : IStarlarkValue, IPaginatedPayload +{ + [JsonPropertyName("total_count")] + public int TotalCount { get; set; } + + [JsonPropertyName("check_suites")] + public PaginatedList CheckSuitesList { get; set; } = new(); + + public CheckSuites() + { + } + + private CheckSuites(int totalCount, PaginatedList checkSuites) + { + CheckSuitesList = checkSuites; + TotalCount = totalCount; + } + + public PaginatedList GetPayload() => CheckSuitesList; + + public IPaginatedPayload AnnotatePayload(string apiPrefix, string? linkHeader) => + new CheckSuites(TotalCount, CheckSuitesList.WithPaginationInfo(apiPrefix, linkHeader)); +} diff --git a/src/Copybara.Core/Git/GitHub/Api/ClientError.cs b/src/Copybara.Core/Git/GitHub/Api/ClientError.cs new file mode 100644 index 000000000..367d695ac --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/ClientError.cs @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Copybara.Git.GitHub.Api; + +/// +/// GitHub client errors always have a message and sometimes a documentation_url. +/// +public class ClientError +{ + [JsonPropertyName("message")] + public string? Message { get; set; } + + [JsonPropertyName("documentation_url")] + public string? DocumentationUrl { get; set; } + + [JsonPropertyName("errors")] + public List? Errors { get; set; } + + public string? GetMessage() => Message; + + public string? GetDocumentationUrl() => DocumentationUrl; + + public IReadOnlyList GetErrors() => + Errors == null ? ImmutableArray.Empty : Errors.ToImmutableArray(); + + public override string ToString() + { + try + { + return JsonSerializer.Serialize(this); + } + catch (Exception e) + { + throw new InvalidOperationException("Unexpected error: ", e); + } + } + + /// An individual error entry within a . + public class ErrorItem + { + [JsonPropertyName("resource")] + public string? Resource { get; set; } + + [JsonPropertyName("field")] + public string? Field { get; set; } + + [JsonPropertyName("code")] + public ErrorType Code { get; set; } + + [JsonPropertyName("message")] + public string? Message { get; set; } + + public string? GetResource() => Resource; + + public string? GetField() => Field; + + public ErrorType GetCode() => Code; + + public string? GetMessage() => Message; + } +} + +/// Error type codes reported by GitHub client errors. +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ErrorType +{ + [JsonStringEnumMemberName("missing")] + MISSING, + + [JsonStringEnumMemberName("missing_field")] + MISSING_FIELD, + + [JsonStringEnumMemberName("invalid")] + INVALID, + + [JsonStringEnumMemberName("already_exists")] + ALREADY_EXISTS, + + [JsonStringEnumMemberName("custom")] + CUSTOM, +} diff --git a/src/Copybara.Core/Git/GitHub/Api/CombinedStatus.cs b/src/Copybara.Core/Git/GitHub/Api/CombinedStatus.cs new file mode 100644 index 000000000..e9bf20137 --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/CombinedStatus.cs @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GitHub.Api; + +/// +/// A combined commit status object. +/// +/// https://developer.github.com/v3/repos/statuses +/// +[StarlarkBuiltin( + "github_api_combined_status_obj", + Doc = + "Combined Information about a commit status as defined in" + + " https://developer.github.com/v3/repos/statuses. This is a subset of the available" + + " fields in GitHub")] +public class CombinedStatus : IStarlarkValue +{ + [JsonPropertyName("state")] + public StatusState State { get; set; } + + [JsonPropertyName("sha")] + public string? Sha { get; set; } + + [JsonPropertyName("total_count")] + public int TotalCount { get; set; } + + [JsonPropertyName("statuses")] + public List? Statuses { get; set; } + + public StatusState GetState() => State; + + [StarlarkMethod( + "state", + Doc = "The overall state of all statuses for a commit: success, failure, pending or error", + StructField = true)] + public string GetStateForSkylark() => State.ToString().ToLowerInvariant(); + + [StarlarkMethod("sha", Doc = "The SHA-1 of the commit", StructField = true)] + public string? GetSha() => Sha; + + [StarlarkMethod("total_count", Doc = "Total number of statuses", StructField = true)] + public int GetTotalCount() => TotalCount; + + [StarlarkMethod("statuses", Doc = "List of statuses for the commit", StructField = true)] + public ISequence GetStatuses() => + StarlarkList.ImmutableCopyOf((Statuses ?? new List()).Cast()); +} diff --git a/src/Copybara.Core/Git/GitHub/Api/CommentBody.cs b/src/Copybara.Core/Git/GitHub/Api/CommentBody.cs new file mode 100644 index 000000000..9b14720f4 --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/CommentBody.cs @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2020 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace Copybara.Git.GitHub.Api; + +/// +/// Request type for https://docs.github.com/en/rest/reference/issues#create-an-issue-comment. +/// +public class CommentBody +{ + [JsonPropertyName("body")] + public string? Body { get; set; } + + public CommentBody(string? body) + { + Body = body; + } + + public CommentBody() + { + } + + public string? GetBody() => Body; +} diff --git a/src/Copybara.Core/Git/GitHub/Api/Commit.cs b/src/Copybara.Core/Git/GitHub/Api/Commit.cs new file mode 100644 index 000000000..00f9369e5 --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/Commit.cs @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GitHub.Api; + +/// Represents the current status of a ref, as returned by the git/refs API call. +[StarlarkBuiltin( + "github_api_commit_obj", + Doc = + "Commit field for GitHub commit information" + + " https://developer.github.com/v3/git/commits/#get-a-commit." + + " This is a subset of the available fields in GitHub")] +public class Commit : IStarlarkValue +{ + [JsonPropertyName("message")] + public string? Message { get; set; } + + [JsonPropertyName("author")] + public CommitAuthor? Author { get; set; } + + [JsonPropertyName("committer")] + public CommitAuthor? Committer { get; set; } + + [StarlarkMethod("message", Doc = "Message of the commit", StructField = true)] + public string? GetMessage() => Message; + + [StarlarkMethod("author", Doc = "Author of the commit", StructField = true)] + public CommitAuthor? GetAuthor() => Author; + + [StarlarkMethod("committer", Doc = "Committer of the commit", StructField = true)] + public CommitAuthor? GetCommitter() => Committer; + + [StarlarkBuiltin( + "github_api_commit_author_obj", + Doc = + "Author/Committer for commit field for GitHub commit information" + + " https://developer.github.com/v3/git/commits/#get-a-commit." + + " This is a subset of the available fields in GitHub")] + public class CommitAuthor : IStarlarkValue + { + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("email")] + public string? Email { get; set; } + + [JsonPropertyName("date")] + public string? Date { get; set; } + + public DateTimeOffset GetDate() => DateTimeOffset.Parse(Date!); + + [StarlarkMethod("date", Doc = "Date of the commit", StructField = true)] + public string? GetDateForSkylark() => Date; + + [StarlarkMethod("name", Doc = "Name of the author/committer", StructField = true)] + public string? GetName() => Name; + + [StarlarkMethod("email", Doc = "Email of the author/committer", StructField = true)] + public string? GetEmail() => Email; + + public override string ToString() => + $"CommitAuthor{{name={Name}, email={Email}, date={Date}}}"; + } + + public override string ToString() => + $"Commit{{message={Message}, author={Author}, committer={Committer}}}"; +} diff --git a/src/Copybara.Core/Git/GitHub/Api/CommitHistoryResponse.cs b/src/Copybara.Core/Git/GitHub/Api/CommitHistoryResponse.cs new file mode 100644 index 000000000..5bcfa1770 --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/CommitHistoryResponse.cs @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; + +namespace Copybara.Git.GitHub.Api; + +/// POJO representing the response from GitHubGraphQLApi.GET_COMMIT_HISTORY_QUERY. +public class CommitHistoryResponse +{ + [JsonPropertyName("data")] + public CommitHistoryData? Data { get; set; } + + public CommitHistoryData? GetData() => Data; + + public override string ToString() => $"CommitHistoryResponse{{data={Data}}}"; + + /// Represents 'data' values. + public class CommitHistoryData + { + [JsonPropertyName("repository")] + public CommitHistoryRepository? Repository { get; set; } + + public CommitHistoryRepository? GetRepository() => Repository; + + public override string ToString() => $"Data{{repository={Repository}}}"; + } + + /// Represents 'repository' values. + public class CommitHistoryRepository + { + [JsonPropertyName("ref")] + public CommitHistoryRef? Ref { get; set; } + + public CommitHistoryRef? GetRef() => Ref; + + public override string ToString() => $"Repository{{ref={Ref}}}"; + } + + /// Represents 'ref' values. + public class CommitHistoryRef + { + [JsonPropertyName("target")] + public Target? Target { get; set; } + + public Target? GetTarget() => Target; + + public override string ToString() => $"Ref{{target={Target}}}"; + } + + /// Represents 'target' values. + public class Target + { + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("history")] + public HistoryNodes? HistoryNodes { get; set; } + + public string? GetId() => Id; + + public HistoryNodes? GetHistoryNodes() => HistoryNodes; + + public override string ToString() => $"Target{{id={Id}, historyNodes={HistoryNodes}}}"; + } + + /// Represents 'history.nodes' values. + public class HistoryNodes + { + [JsonPropertyName("nodes")] + public List? Nodes { get; set; } + + public List? GetNodes() => Nodes; + + public override string ToString() => $"HistoryNodes{{nodes={Nodes}}}"; + } + + /// Represents 'history.node' element values. + public class HistoryNode + { + [JsonPropertyName("associatedPullRequests")] + public AssociatedPullRequests? AssociatedPullRequests { get; set; } + + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("oid")] + public string? Oid { get; set; } + + public AssociatedPullRequests? GetAssociatedPullRequests() => AssociatedPullRequests; + + public string? GetId() => Id; + + public string? GetOid() => Oid; + + public override string ToString() => + $"HistoryNode{{associatedPullRequest={AssociatedPullRequests}, id={Id}, oid={Oid}}}"; + } + + /// Represents 'associatedPullRequests' values. + public class AssociatedPullRequests + { + [JsonPropertyName("edges")] + public List? Edges { get; set; } + + public List? GetEdges() => Edges; + + public override string ToString() => $"AssociatedPullRequests{{edges={Edges}}}"; + } + + /// Represents 'associatedPullRequests.edges' values. + public class PullRequestEdges + { + [JsonPropertyName("node")] + public AssociatedPullRequestNode? Node { get; set; } + + public AssociatedPullRequestNode? GetNode() => Node; + + public override string ToString() => $"PullRequestEdges{{node={Node}}}"; + } + + /// Represents 'associatedPullRequests.edges.node' values. + public class AssociatedPullRequestNode + { + [JsonPropertyName("reviewDecision")] + public string? ReviewDecision { get; set; } + + [JsonPropertyName("author")] + public Author? Author { get; set; } + + [JsonPropertyName("latestOpinionatedReviews")] + public LatestOpinionatedReviews? LatestOpinionatedReviews { get; set; } + + [JsonPropertyName("mergedBy")] + public MergedBy? MergedBy { get; set; } + + [JsonPropertyName("title")] + public string? Title { get; set; } + + public string? GetReviewDecision() => ReviewDecision; + + public Author? GetAuthor() => Author; + + public LatestOpinionatedReviews? GetLatestOpinionatedReviews() => LatestOpinionatedReviews; + + public MergedBy? GetMergedBy() => MergedBy; + + public string? GetTitle() => Title; + + public override string ToString() => + $"AssociatedPullRequestNode{{reviewDecision={ReviewDecision}, author={Author}," + + $" latestOpinionatedReviews={LatestOpinionatedReviews}, mergedBy={MergedBy}," + + $" title={Title}}}"; + } + + /// Represents 'latestOpinionatedReviews' values. + public class LatestOpinionatedReviews + { + [JsonPropertyName("edges")] + public List? Edges { get; set; } + + public List? GetEdges() => Edges; + + public override string ToString() => $"LatestOpinionatedReviews{{edges={Edges}}}"; + } + + /// Represents 'latestOptionatedReviews.edges' values. + public class AuthorEdges + { + [JsonPropertyName("node")] + public AuthorNode? Node { get; set; } + + public AuthorNode? GetNode() => Node; + + public override string ToString() => $"AuthorEdges{{node={Node}}}"; + } + + /// Represents 'latestOpinionatedReviews.edges.node' values. + public class AuthorNode + { + [JsonPropertyName("author")] + public Author? Author { get; set; } + + [JsonPropertyName("state")] + public string? State { get; set; } + + public string? GetState() => State; + + public Author? GetAuthor() => Author; + + public override string ToString() => $"AuthorNode{{author={Author}, state={State}}}"; + } + + /// Represents 'author' values. + public class Author + { + [JsonPropertyName("login")] + public string? Login { get; set; } + + public string? GetLogin() => Login; + + public override string ToString() => $"Author{{login={Login}}}"; + } + + /// Represents 'mergedBy' values. + public class MergedBy + { + [JsonPropertyName("login")] + public string? Login { get; set; } + + public string? GetLogin() => Login; + + public override string ToString() => $"MergedBy{{login={Login}}}"; + } +} diff --git a/src/Copybara.Core/Git/GitHub/Api/CreatePullRequest.cs b/src/Copybara.Core/Git/GitHub/Api/CreatePullRequest.cs new file mode 100644 index 000000000..aca3ff51e --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/CreatePullRequest.cs @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Copybara.Common; + +namespace Copybara.Git.GitHub.Api; + +/// An object that represents the creation of a Pull Request. +public class CreatePullRequest +{ + [JsonPropertyName("title")] + public string? Title { get; set; } + + [JsonPropertyName("body")] + public string? Body { get; set; } + + /// + /// Branch to use for the pull request, can be a reference to another github repository if + /// somerepo:branch format is used. + /// + [JsonPropertyName("head")] + public string? Head { get; set; } + + /// Base of the pull request, usually something like 'master'. + [JsonPropertyName("base")] + public string? Base { get; set; } + + [JsonPropertyName("draft")] + public bool Draft { get; set; } + + public string? GetTitle() => Title; + + public string? GetBody() => Body; + + public string? GetHead() => Head; + + public string? GetBase() => Base; + + public void SetTitle(string title) => Title = title; + + public bool GetDraft() => Draft; + + public CreatePullRequest(string title, string body, string head, string @base, bool draft) + { + Title = Preconditions.CheckNotNull(title); + Body = Preconditions.CheckNotNull(body); + Head = Preconditions.CheckNotNull(head); + Base = Preconditions.CheckNotNull(@base); + Draft = draft; + } + + public CreatePullRequest() + { + } +} diff --git a/src/Copybara.Core/Git/GitHub/Api/CreateReleaseRequest.cs b/src/Copybara.Core/Git/GitHub/Api/CreateReleaseRequest.cs new file mode 100644 index 000000000..33f261dcc --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/CreateReleaseRequest.cs @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Starlark.Annot; +using Starlark.Eval; + +namespace Copybara.Git.GitHub.Api; + +/// +/// Input for creating a release. +/// https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#create-a-release +/// +[StarlarkBuiltin( + "github_create_release_obj", + Doc = "GitHub API value type for release params. See " + + "https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#create-a-release")] +public class CreateReleaseRequest : IStarlarkValue +{ + [JsonPropertyName("tag_name")] + public string? TagName { get; set; } + + [JsonPropertyName("body")] + public string? Body { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("target_commitish")] + public string? TargetCommitish { get; set; } + + [JsonPropertyName("prerelease")] + public bool? PreRelease { get; set; } + + [JsonPropertyName("draft")] + public bool? Draft { get; set; } + + [JsonPropertyName("make_latest")] + public bool? MakeLatest { get; set; } + + [JsonPropertyName("generate_release_notes")] + public bool? GenerateReleaseNotes { get; set; } + + public CreateReleaseRequest(string tagName) + { + TagName = tagName; + } + + public CreateReleaseRequest() + { + // just for reflection. + } + + [StarlarkMethod( + "with_body", + Doc = "Set the body for the release.")] + public CreateReleaseRequest WithBody([Param(Name = "body", Doc = "Body for the release")] string body) + { + Body = body; + return this; + } + + [StarlarkMethod( + "with_name", + Doc = "Set the name for the release.")] + public CreateReleaseRequest WithName([Param(Name = "name", Doc = "Name for the release")] string name) + { + Name = name; + return this; + } + + [StarlarkMethod( + "with_commitish", + Doc = "Set the commitish to be used for the release. Defaults to HEAD")] + public CreateReleaseRequest WithCommitish( + [Param(Name = "commitish", Doc = "Commitish for the release")] string targetCommitish) + { + TargetCommitish = targetCommitish; + return this; + } + + [StarlarkMethod( + "set_draft", + Doc = "Is this a draft release?")] + public CreateReleaseRequest WithDraft([Param(Name = "draft", Doc = "Mark release as draft?")] bool draft) + { + Draft = draft; + return this; + } + + [StarlarkMethod( + "set_latest", + Doc = "Is this the latest release?")] + public CreateReleaseRequest WithMakeLatest( + [Param(Name = "make_latest", Doc = "Mark release as latest?")] bool makeLatest) + { + MakeLatest = makeLatest; + return this; + } + + [StarlarkMethod( + "set_prerelease", + Doc = "Is this a prerelease?")] + public CreateReleaseRequest WithPreRelease( + [Param(Name = "prerelease", Doc = "Mark release as prerelease?")] bool preRelease) + { + PreRelease = preRelease; + return this; + } + + [StarlarkMethod( + "set_generate_release_notes", + Doc = "Generate release notes?")] + public CreateReleaseRequest WithGenerateReleaseNotes( + [Param(Name = "generate_notes", Doc = "Generate notes?")] bool generateReleaseNotes) + { + GenerateReleaseNotes = generateReleaseNotes; + return this; + } + + public string? GetTagName() => TagName; + + public string? GetBody() => Body; + + public string? GetName() => Name; + + public string? GetTargetCommitish() => TargetCommitish; +} diff --git a/src/Copybara.Core/Git/GitHub/Api/CreateStatusRequest.cs b/src/Copybara.Core/Git/GitHub/Api/CreateStatusRequest.cs new file mode 100644 index 000000000..4f80dec6e --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/CreateStatusRequest.cs @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text.Json.Serialization; +using Copybara.Common; + +namespace Copybara.Git.GitHub.Api; + +/// +/// Request for creating commit statuses: +/// +/// https://developer.github.com/v3/repos/statuses/#create-a-status +/// +public class CreateStatusRequest +{ + [JsonPropertyName("state")] + public StatusState State { get; set; } + + [JsonPropertyName("target_url")] + public string? TargetUrl { get; set; } + + [JsonPropertyName("description")] + public string? Description { get; set; } + + [JsonPropertyName("context")] + public string Context { get; set; } + + public CreateStatusRequest( + StatusState state, string? targetUrl, string? description, string context) + { + State = state; + TargetUrl = targetUrl; + Description = description; + Context = Preconditions.CheckNotNull(context); + } + + public StatusState GetState() => State; + + public string? GetTargetUrl() => TargetUrl; + + public string? GetDescription() => Description; + + public string GetContext() => Context; +} diff --git a/src/Copybara.Core/Git/GitHub/Api/GitHubApi.cs b/src/Copybara.Core/Git/GitHub/Api/GitHubApi.cs new file mode 100644 index 000000000..a42b4a05a --- /dev/null +++ b/src/Copybara.Core/Git/GitHub/Api/GitHubApi.cs @@ -0,0 +1,674 @@ +/* + * Copyright (C) 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Immutable; +using System.Text; +using Copybara.Common; +using Copybara.Exceptions; +using ConsoleT = Copybara.Util.Console.Console; +using ProfilerT = Copybara.Profiler.Profiler; + +namespace Copybara.Git.GitHub.Api; + +/// +/// A mini API for getting and updating GitHub projects through the GitHub REST API. +/// +public class GitHubApi +{ + private readonly IGitHubApiTransport _transport; + private readonly ProfilerT _profiler; + private readonly ConsoleT? _console; + + public const int MaxPerPage = 100; + private const int MaxPages = 10; + + public GitHubApi(IGitHubApiTransport transport, ProfilerT profiler) + : this(transport, profiler, null) + { + } + + public GitHubApi(IGitHubApiTransport transport, ProfilerT profiler, ConsoleT? console) + { + _transport = Preconditions.CheckNotNull(transport); + _profiler = Preconditions.CheckNotNull(profiler); + _console = console; + } + + /// Get all the pull requests for a project. + /// a project in the form of "google/copybara" + /// query parameters for the list operation + public Task> GetPullRequestsAsync( + string projectId, PullRequestListParams @params) + { + Preconditions.CheckNotNull(@params); + return PaginatedGetAsync>( + "github_api_list_pulls", + "Project", + ImmutableListMultimap.Empty, + string.Format( + "repos/{0}/pulls?per_page={1}{2}", projectId, MaxPerPage, @params.ToParams()), + "GET repos/%s/pulls"); + } + + /// + /// Get a specific pull request for a project. + /// + public async Task GetPullRequestAsync(string projectId, long number) + { + using ProfilerTaskScope ignore = Scope("github_api_get_pull"); + try + { + return (await _transport.GetAsync( + $"repos/{projectId}/pulls/{number}", "GET repos/%s/pulls/%d").ConfigureAwait(false))!; + } + catch (GitHubApiException e) + { + throw TreatGitHubException(e, "Pull Request"); + } + } + + /// Get comments for a specific pull request. + public async Task GetPullRequestCommentAsync(string projectId, long commentId) + { + using ProfilerTaskScope ignore = Scope("github_api_get_pull_comment"); + try + { + return (await _transport.GetAsync( + $"repos/{projectId}/pulls/comments/{commentId}", + "GET repos/%s/pulls/comments/%d").ConfigureAwait(false))!; + } + catch (GitHubApiException e) + { + throw TreatGitHubException(e, "Pull Request Comment"); + } + } + + /// Get comments for a specific pull request. + public Task> GetPullRequestCommentsAsync( + string projectId, long prNumber) + { + return PaginatedGetAsync>( + "github_api_get_reviews", + "Pull Request Comments", + ImmutableListMultimap.Empty, + $"repos/{projectId}/pulls/{prNumber}/comments?per_page={MaxPerPage}", + "GET repos/%s/pulls/%d/comments"); + } + + /// Get reviews for a pull request. + public Task> GetReviewsAsync(string projectId, long number) + { + return PaginatedGetAsync>( + "github_api_get_reviews", + "Pull Request or project", + ImmutableListMultimap.Empty, + $"repos/{projectId}/pulls/{number}/reviews?per_page={MaxPerPage}", + "GET repos/%s/pulls/%d/reviews"); + } + + private async Task> PaginatedGetAsync( + string profilerName, + string entity, + ImmutableListMultimap headers, + string path, + string requestTemplate) + where TResponse : class, IPaginatedPayload + { + var builder = ImmutableArray.CreateBuilder(); + int pages = 0; + string? currentPath = path; + while (currentPath != null && pages < MaxPages) + { + using ProfilerTaskScope ignore = Scope($"{profilerName}_page_{pages}"); + try + { + TResponse? response = + await _transport.GetAsync(currentPath, headers, requestTemplate) + .ConfigureAwait(false); + PaginatedList page = response!.GetPayload(); + builder.AddRange(page.GetElements()); + currentPath = page.NextUrl; + pages++; + } + catch (GitHubApiException e) + { + throw TreatGitHubException(e, entity); + } + } + + if (pages == MaxPages && _console != null && currentPath != null) + { + _console.WarnFmt( + "Copybara ran a paginated GET request {0} to GitHub for {1} pages, and that is the" + + " maximum number of pages Copybara will read. It is possible that additional pages" + + " were not read.", + currentPath, MaxPages); + } + + return builder.ToImmutable(); + } + + /// Create a pull request. + public async Task CreatePullRequestAsync(string projectId, CreatePullRequest request) + { + using ProfilerTaskScope ignore = Scope("github_api_create_pull"); + return (await _transport.PostAsync( + $"repos/{projectId}/pulls", request, "POST repos/%s/pulls").ConfigureAwait(false))!; + } + + /// + /// Adds assignees to an issue or pull request. + /// https://docs.github.com/en/rest/issues/assignees#add-assignees-to-an-issue + /// + public async Task AddAssigneesAsync(string projectId, long number, AddAssignees request) + { + using ProfilerTaskScope ignore = Scope("github_api_add_assignees"); + return (await _transport.PostAsync( + $"repos/{projectId}/issues/{number}/assignees", request, + "POST repos/%s/issues/%d/assignees").ConfigureAwait(false))!; + } + + /// Update a pull request. + public async Task UpdatePullRequestAsync( + string projectId, long number, UpdatePullRequest request) + { + using ProfilerTaskScope ignore = Scope("github_api_update_pull"); + return (await _transport.PostAsync( + $"repos/{projectId}/pulls/{number}", request, "POST repos/%s/pulls/%s") + .ConfigureAwait(false))!; + } + + /// + /// Listing issues and pull requests based on . + /// https://docs.github.com/en/rest/search#search-issues-and-pull-requests + /// + public async Task GetIssuesOrPullRequestsSearchResultsAsync( + IssuesAndPullRequestsSearchRequestParams @params) + { + using ProfilerTaskScope ignore = Scope("github_api_search_issues_or_pull_requests"); + return (await _transport.GetAsync( + $"search/issues?q={@params.ToParams()}", "GET search/issues?q=%s").ConfigureAwait(false))!; + } + + /// + /// Get a user's permission level. + /// https://developer.github.com/v3/repos/collaborators/#review-a-users-permission-level + /// + public async Task GetUserPermissionLevelAsync( + string projectId, string usrLogin) + { + using ProfilerTaskScope ignore = Scope("github_api_update_pull"); + return (await _transport.GetAsync( + $"repos/{projectId}/collaborators/{usrLogin}/permission", + "GET repos/%s/collaborators/%s/permission").ConfigureAwait(false))!; + } + + /// + /// Get authenticated User. https://developer.github.com/v3/users/#get-the-authenticated-user + /// + public async Task GetAuthenticatedUserAsync() + { + using ProfilerTaskScope ignore = Scope("github_api_get_authenticated_user"); + return (await _transport.GetAsync("user", "GET user").ConfigureAwait(false))!; + } + + /// + /// Get a specific issue for a project. + /// + /// Use this method to get the Pull Request labels. + /// + public async Task GetIssueAsync(string projectId, long number) + { + using ProfilerTaskScope ignore = Scope("github_api_get_issue"); + try + { + return (await _transport.GetAsync( + $"repos/{projectId}/issues/{number}", "GET repos/%s/issues/%d") + .ConfigureAwait(false))!; + } + catch (GitHubApiException e) + { + throw TreatGitHubException(e, "Issue"); + } + } + + /// Create an issue. + public async Task CreateIssueAsync(string projectId, Issue.CreateIssueRequest request) + { + using ProfilerTaskScope ignore = Scope("github_api_create_issue"); + return (await _transport.PostAsync( + $"repos/{projectId}/issues", request, "POST repos/%s/issues").ConfigureAwait(false))!; + } + + /// Get all the refs for a repo (git ls-remote). + public async Task> GetLsRemoteAsync(string projectId) + { + using ProfilerTaskScope ignore = Scope("github_api_list_refs"); + try + { + List? result = await _transport.GetAsync>( + $"repos/{projectId}/git/refs?per_page={MaxPerPage}", "GET repos/%s/git/refs") + .ConfigureAwait(false); + return (result ?? new List()).ToImmutableArray(); + } + catch (GitHubApiException e) + { + // Per https://developer.github.com/v3/git/, GH returns 409 - conflict if the repo is + // empty or in the process of being created. + if (e.GetResponseCode() == GitHubApiResponseCode.CONFLICT) + { + return ImmutableArray.Empty; + } + + throw; + } + } + + public async Task CreateStatusAsync( + string projectId, string sha1, CreateStatusRequest request) + { + using ProfilerTaskScope ignore = Scope("github_api_create_status"); + Status result = (await _transport.PostAsync( + $"repos/{projectId}/statuses/{sha1}", request, "Create status").ConfigureAwait(false))!; + if (result.GetContext() == null || result.StateRaw == null) + { + throw new RepoException( + "Something went wrong at the GitHub API transport level." + + $" Context: {result.GetContext()} state: {result.StateRaw}"); + } + + return result; + } + + public async Task UpdateReferenceAsync( + string projectId, string @ref, UpdateReferenceRequest request) + { + Preconditions.CheckArgument( + @ref.StartsWith("refs/", StringComparison.Ordinal), + "References has to be complete references in the form of refs/heads/foo. But was: {0}", + @ref); + using ProfilerTaskScope ignore = Scope("github_api_update_reference"); + Ref result = (await _transport.PostAsync( + $"repos/{projectId}/git/{@ref}", request, "POST repos/%s/git/%s").ConfigureAwait(false))!; + if (result.GetRef() == null || result.GetSha() == null || result.GetUrl() == null) + { + throw new RepoException( + "Something went wrong at the GitHub API transport level." + + $" ref: {result.GetRef()} sha: {result.GetSha()}, url: {result.GetUrl()}"); + } + + return result; + } + + public async Task DeleteReferenceAsync(string projectId, string @ref) + { + Preconditions.CheckArgument( + @ref.StartsWith("refs/", StringComparison.Ordinal), + "References has to be complete references in the form of refs/heads/foo. But was: {0}", + @ref); + // There is no good reason for deleting master. + Preconditions.CheckArgument( + @ref != "refs/heads/master", + "Copybara doesn't allow to delete master branch for security reasons"); + + using ProfilerTaskScope ignore = Scope("github_api_delete_reference"); + await _transport.DeleteAsync($"repos/{projectId}/git/{@ref}", "DELETE repos/%s/git/%s") + .ConfigureAwait(false); + } + + public async Task GetReferenceAsync(string projectId, string @ref) + { + using ProfilerTaskScope ignore = Scope("github_api_get_reference"); + ValidationException.CheckCondition( + @ref.StartsWith("refs/", StringComparison.Ordinal), "Ref must start with \"refs/\""); + return (await _transport.GetAsync( + $"repos/{projectId}/git/{@ref}", "GET repos/%s/git/%s").ConfigureAwait(false))!; + } + + public async Task GetRepositoryAsync(string projectId) + { + using ProfilerTaskScope ignore = Scope("github_api_get_repository"); + return (await _transport.GetAsync( + $"repos/{projectId}", "GET repos/%s").ConfigureAwait(false))!; + } + + public Task> GetReferencesAsync(string projectId) + { + using ProfilerTaskScope ignore = Scope("github_api_get_references"); + return PaginatedGetAsync>( + "github_api_get_references", + "Project", + ImmutableListMultimap.Empty, + $"repos/{projectId}/git/refs?per_page={MaxPerPage}", + "GET repos/%s/git/refs"); + } + + public async Task GetCombinedStatusAsync(string projectId, string @ref) + { + using ProfilerTaskScope ignore = Scope("github_api_get_combined_status"); + return (await _transport.GetAsync( + $"repos/{projectId}/commits/{@ref}/status?per_page={MaxPerPage}", + "GET repos/%s/commits/%s/status").ConfigureAwait(false))!; + } + + /// + /// Calls the GitHub API REST endpoint to list check runs for a specific ref. + /// + /// If is provided, only check runs with that name are + /// returned. + /// + public Task> GetCheckRunsAsync( + string projectId, string @ref, string? checkName) + { + using ProfilerTaskScope ignore = Scope("github_api_get_check_runs"); + var headers = ImmutableListMultimap.Of( + "Accept", "application/vnd.github.antiope-preview+json"); + if (string.IsNullOrEmpty(checkName)) + { + return PaginatedGetAsync( + "github_api_get_check_runs_get", + "Check Run", + headers, + $"repos/{projectId}/commits/{@ref}/check-runs?per_page={MaxPerPage}", + "GET repos/%s/commits/%s/check-runs"); + } + + return PaginatedGetAsync( + "github_api_get_check_runs_get", + "Check Run", + headers, + $"repos/{projectId}/commits/{@ref}/check-runs?per_page={MaxPerPage}&check_name={checkName}", + "GET repos/%s/commits/%s/check-runs"); + } + + /// + /// Calls the GitHub API REST endpoint to list check runs for a specific ref. + /// + public Task> GetCheckRunsAsync(string projectId, string @ref) => + GetCheckRunsAsync(projectId, @ref, checkName: null); + + /// + /// https://docs.github.com/en/rest/checks/suites/#list-check-suites-for-a-git-reference + /// + public Task> GetCheckSuitesAsync(string projectId, string @ref) + { + using ProfilerTaskScope ignore = Scope("github_api_get_check_suites"); + var headers = ImmutableListMultimap.Of( + "Accept", "application/vnd.github.antiope-preview+json"); + return PaginatedGetAsync( + "github_api_get_check_runs_get", + "Check Run", + headers, + $"repos/{projectId}/commits/{@ref}/check-suites?per_page={MaxPerPage}", + "GET repos/%s/commits/%s/check-suites"); + } + + public async Task GetCommitAsync(string projectId, string @ref) + { + using ProfilerTaskScope ignore = Scope("github_api_get_commit"); + return (await _transport.GetAsync( + $"repos/{projectId}/commits/{@ref}", "GET repos/%s/commits/%s").ConfigureAwait(false))!; + } + + /// https://developer.github.com/v3/issues/labels/#add-labels-to-an-issue + public async Task> AddLabelsAsync( + string project, long prNumber, IReadOnlyList labels) + { + using ProfilerTaskScope ignore = Scope("github_api_add_labels"); + List