diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000000..2f76be5dfa8 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,20 @@ +# Make sure RUBY_VERSION matches the Ruby version in .ruby-version +ARG RUBY_VERSION=3.4.7 +FROM ghcr.io/rails/devcontainer/images/ruby:$RUBY_VERSION + +USER root +ENV DEBIAN_FRONTEND=noninteractive + +# Required on Windows +RUN mkdir -p /workspaces/openstreetmap-website \ + && chown -R vscode:vscode /workspaces/openstreetmap-website + +# Install system packages then clean up to minimize image size +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + file \ + libarchive-dev \ + libgd-dev \ + osmosis \ + && apt-get clean \ + && apt-get distclean diff --git a/.devcontainer/compose.yaml b/.devcontainer/compose.yaml new file mode 100644 index 00000000000..7cb518323aa --- /dev/null +++ b/.devcontainer/compose.yaml @@ -0,0 +1,53 @@ +name: "open_street_map" + +services: + rails-app: + build: + context: .. + dockerfile: .devcontainer/Dockerfile + + volumes: + - ../..:/workspaces:cached + - mise-cache:/home/vscode/.local/share/mise + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + # Uncomment the next line to use a non-root user for all processes. + # user: vscode + + # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + depends_on: + - selenium-default + - selenium-de + - selenium-nolang + - postgres + + selenium-default: + image: selenium/standalone-firefox + restart: unless-stopped + + selenium-de: + image: selenium/standalone-firefox + restart: unless-stopped + + selenium-nolang: + image: selenium/standalone-firefox + restart: unless-stopped + + postgres: + image: postgres:16.1 + restart: unless-stopped + networks: + - default + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + POSTGRES_DB: openstreetmap + POSTGRES_USER: openstreetmap + POSTGRES_PASSWORD: openstreetmap + +volumes: + postgres-data: + mise-cache: diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000000..88a00f971e6 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,34 @@ +// For format details, see https://containers.dev/implementors/json_reference/. +// For config options, see the README at: https://github.com/devcontainers/templates/tree/main/src/ruby +{ + "name": "open_street_map", + "dockerComposeFile": "compose.yaml", + "service": "rails-app", + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", + + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/node:1": {}, + "ghcr.io/rails/devcontainer/features/activestorage": {}, + "ghcr.io/rails/devcontainer/features/postgres-client": {} + }, + + "containerEnv": { + "CAPYBARA_SERVER_PORT": "45678", + "DB_HOST": "postgres" + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [3000, 5432], + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as root instead. More info: https://containers.dev/implementors/json_reference/#remoteUser. + // "remoteUser": "root", + + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": ".devcontainer/start" +} diff --git a/.devcontainer/start b/.devcontainer/start new file mode 100755 index 00000000000..9b4d529c7d4 --- /dev/null +++ b/.devcontainer/start @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +# This setup uses https://mise.jdx.dev to manage language versions. +# Mark the config file as trusted to avoid an interactive prompt. +mise trust /workspaces/mise.local.toml + +cp config/devcontainer.database.yml config/database.yml +cp config/example.storage.yml config/storage.yml +touch config/settings.local.yml + +bin/setup --skip-server diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml new file mode 100644 index 00000000000..de0d1f6e689 --- /dev/null +++ b/.github/workflows/devcontainer.yml @@ -0,0 +1,21 @@ +name: devcontainer + +on: + - push + - pull_request + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout (GitHub) + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Build and run dev container task + uses: devcontainers/ci@8bf61b26e9c3a98f69cb6ce2f88d24ff59b785c6 # v0.3.1900000417 + with: + runCmd: bin/rails test:all diff --git a/DEVCONTAINER.md b/DEVCONTAINER.md new file mode 100644 index 00000000000..10b02513342 --- /dev/null +++ b/DEVCONTAINER.md @@ -0,0 +1,141 @@ +# Using Development Containers for development and testing + +You can set up a development environment for this project using [Development Containers](https://containers.dev/), aka devcontainers. + +There are different ways of working with devcontainers. Some are automated and integrated in IDE applications, while others are more manual and require CLI tools. This guide will use [Visual Studio Code](https://code.visualstudio.com) and assumes that it's installed already. + +Be aware that the containers will take significant space in your hard drive, in the region of 6GB. + +## Install Git + +You will need Git to clone (download) the code repository and push changes. How to install Git will depend on your operating system. + +### Ubuntu 24.04 LTS, Debian 13 (Trixie) + +```bash +sudo apt-get update +sudo apt-get install git-core +``` + +### Fedora + +```bash +sudo dnf install git +``` + +### macOS + +For macOS, you will need [Xcode Command Line Tools](https://mac.install.guide/commandlinetools/); macOS 14 (Sonoma) or later; and some familiarity with Unix development via the Terminal. + +### Windows + +You can install Git using [WinGet](https://learn.microsoft.com/en-gb/windows/package-manager/winget/): + +```bash +winget install --id Git.Git -e --source winget +``` + +Alternatively, see [other install options](https://git-scm.com/install/windows). + +## Install Docker + +You will need to install Docker in order to run devcontainers. Again, the methods will depend on your operating system. + +In general: if you get an error about not having permission to use Docker, you may need to add your user to the `docker` group. The details vary, but often this is a case of running the following command: + +```bash +sudo usermod --append --group docker $USERNAME +``` + +Substitute `$USERNAME` with your actual username in the command above. After that, reboot your computer to ensure that this change takes effect. + +### Windows + +1. Use Docker Desktop via [docker.com Download](https://www.docker.com/products/docker-desktop/). + +2. You have to enable git symlinks before cloning the repository. + This repository uses symbolic links that are not enabled by default on Windows git. To enable them, [turn on Developer Mode](https://windowsreport.com/windows-11-developer-mode/) on Windows and run `git config --global core.symlinks true` to enable symlinks in Git. See [this StackOverflow question](https://stackoverflow.com/questions/5917249/git-symbolic-links-in-windows) for more information. + +### Mac + +- Use Docker Desktop via [docker.com Download](https://www.docker.com/products/docker-desktop/). +- Or [Homebrew](https://formulae.brew.sh/cask/docker). + +### Linux + +Use [Docker Engine](https://docs.docker.com/engine/install/ubuntu/) with the [docker-compose-plugin](https://docs.docker.com/compose/install/linux/) + +## Clone the repository + +The repository is reasonably large (~560MB) and it's unlikely that you'll need the full history. Therefore you can probably do with a shallow clone (~100MB): +```bash +git clone --depth=1 https://github.com/openstreetmap/openstreetmap-website.git +``` + +If you want to add in the full history later on, perhaps to run `git blame` or `git log`, run `git fetch --unshallow`. + +> [!TIP] +> To download the full history from the start, run: +> ```bash +> git clone https://github.com/openstreetmap/openstreetmap-website.git +> ``` + +## Install Dev Containers extension + +Start by opening the project with VS Code. Within it, you will need to install the extension _Dev Containers_, which can be done from the _Extensions_ section, reachable via the sidebar icons. Or VS Code may show a popup recommending this extension, with a button to install it directly. + +![VS Code: panel to install extensions](./docs/assets/vscode-devcontainers-extension.png) + +## Open the project in a container + +If everything is correct, this should make a few new commands available to you within VS Code. Find and run the command "Dev Containers: Reopen in Container". + +![VS Code: command to open in a devcontainer](./docs/assets/vscode-dev-reopen.png) + +The first time you do this, it will go into a bit of a process. It will create the devcontainer, pull the Docker images, install the dependencies, etc. Go drink some water while this runs. + +![VS Code: notification that VS Code is connecting to the Dev Container](./docs/assets/vscode-connecting-to-devcontainer.png) + +Eventually this will present you with a development environment ready to go. In subsequent occasions this should be much faster. + +## Done! + +If everything went well, you are done! For example, now you can open a shell in this environment using the VS Studio command "Create New Terminal (With Profile)": + +![VS Code: command to open a terminal](./docs/assets/vscode-create-terminal.png) + +From this terminal, you can run the test suite with `bundle exec rails test:all`: + +![Running the test suite in the terminal](./docs/assets/vscode-rails-test-all.png) + +Hopefully it should be all green? 🤞 You can also start a development server with `bundle exec rails s`: + +![Running the dev server in the terminal](./docs/assets/vscode-rails-server.png) + +It will take a moment to start, after which you will be able to take a browser to http://localhost:3000 and visit your own local version of the site. + +## Other useful configuration + +See [`CONFIGURE.md`](CONFIGURE.md) for information on how to manage users and enable OAuth for iD, JOSM etc. + +## Other tools + +VS Code is not the only way to work with devcontainers. Other options include: + +- [RubyMine](https://www.jetbrains.com/help/ruby/start-dev-container-inside-ide.html): another popular environment to work with Ruby. +- [DevPod](https://devpod.sh): a CLI tool to work with devcontainers. +- [VSCodium](https://vscodium.com): a Free/Libre alternative to VS Code. +- [GitHub Codespaces](https://github.com/codespaces): GitHub's hosted IDE. + +## Troubleshooting + +### `‘ruby’: No such file or directory` + +In some cases Ruby may not install correctly. If you see this message, run these two commands: + +``` +mise install +bundle install +``` + +This has been observed particularly when using RubyMine. diff --git a/INSTALL.md b/INSTALL.md index 7bf580b1ed0..1c4abf9182a 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -8,14 +8,17 @@ There is more than one way to set up a development environment. ### Containerized Installation -We offer a containerized environment with Docker which may avoid installation difficulties. See [DOCKER.md](DOCKER.md) for complete instructions. +We offer containerized environments with Docker which may avoid installation difficulties: + +- To use Docker manually, see [DOCKER](DOCKER.md). +- To use Docker via [Development Containers](https://containers.dev) (devcontainers), see [DEVCONTAINER.md](DEVCONTAINER.md). ### Manual Installation Install dependencies directly on your machine (traditional approach, covered in this guide). This gives you the most control and is often preferred by experienced developers on Linux systems. > [!WARNING] -> **Windows Note:** We don't recommend using this approach on Windows, as some Ruby gems may not be supported. If you are using Windows, we recommend containerized setup using [Docker](DOCKER.md). +> **Windows Note:** We don't recommend using this approach on Windows, as some Ruby gems may not be supported. If you are using Windows, we recommend a containerized setup as mentioned above. ## Manual Installation Guide diff --git a/bin/setup b/bin/setup index f882c0024ed..ccac37778a5 100755 --- a/bin/setup +++ b/bin/setup @@ -16,6 +16,7 @@ FileUtils.chdir APP_ROOT do puts "== Installing dependencies ==" system("bundle check") || system!("bundle install") + system("yarn install --check-dependencies") # puts "\n== Copying sample files ==" # unless File.exist?("config/database.yml") diff --git a/config/devcontainer.database.yml b/config/devcontainer.database.yml new file mode 100644 index 00000000000..f65469580c1 --- /dev/null +++ b/config/devcontainer.database.yml @@ -0,0 +1,20 @@ +# This configuration is tailored for use with devcontainers. See DEVCONTAINER.md for more information. + +development: + adapter: postgresql + database: openstreetmap + username: openstreetmap + password: openstreetmap + host: <%= ENV["DB_HOST"] %> + encoding: utf8 + pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + +test: + adapter: postgresql + database: osm_test + username: openstreetmap + password: openstreetmap + host: <%= ENV["DB_HOST"] %> + encoding: utf8 + pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + diff --git a/docs/assets/vscode-connecting-to-devcontainer.png b/docs/assets/vscode-connecting-to-devcontainer.png new file mode 100644 index 00000000000..10666d7f99c Binary files /dev/null and b/docs/assets/vscode-connecting-to-devcontainer.png differ diff --git a/docs/assets/vscode-create-terminal.png b/docs/assets/vscode-create-terminal.png new file mode 100644 index 00000000000..07d1ad61eaf Binary files /dev/null and b/docs/assets/vscode-create-terminal.png differ diff --git a/docs/assets/vscode-dev-reopen.png b/docs/assets/vscode-dev-reopen.png new file mode 100644 index 00000000000..6266d34c0c9 Binary files /dev/null and b/docs/assets/vscode-dev-reopen.png differ diff --git a/docs/assets/vscode-devcontainers-extension.png b/docs/assets/vscode-devcontainers-extension.png new file mode 100644 index 00000000000..976b5cf05f9 Binary files /dev/null and b/docs/assets/vscode-devcontainers-extension.png differ diff --git a/docs/assets/vscode-rails-server.png b/docs/assets/vscode-rails-server.png new file mode 100644 index 00000000000..9b82ba7c95e Binary files /dev/null and b/docs/assets/vscode-rails-server.png differ diff --git a/docs/assets/vscode-rails-test-all.png b/docs/assets/vscode-rails-test-all.png new file mode 100644 index 00000000000..24b1028db98 Binary files /dev/null and b/docs/assets/vscode-rails-test-all.png differ diff --git a/docs/assets/vscode-reopen-devcontainer.png b/docs/assets/vscode-reopen-devcontainer.png new file mode 100644 index 00000000000..329cc5be457 Binary files /dev/null and b/docs/assets/vscode-reopen-devcontainer.png differ diff --git a/docs/assets/vscode-terminal-ready.png b/docs/assets/vscode-terminal-ready.png new file mode 100644 index 00000000000..fb1a8be10e9 Binary files /dev/null and b/docs/assets/vscode-terminal-ready.png differ diff --git a/test/application_system_test_case.rb b/test/application_system_test_case.rb index e10501f2b71..be7b35f0cac 100644 --- a/test/application_system_test_case.rb +++ b/test/application_system_test_case.rb @@ -15,11 +15,41 @@ class ApplicationSystemTestCase < ActionDispatch::SystemTestCase config.enable_aria_label = true end - driven_by :selenium, :using => Settings.system_test_headless ? :headless_firefox : :firefox do |options| - options.add_preference("intl.accept_languages", "en") - options.binary = Settings.system_test_firefox_binary if Settings.system_test_firefox_binary + cattr_accessor(:capybara_server_port) { ENV.fetch("CAPYBARA_SERVER_PORT", nil) } + + served_by :host => "rails-app", :port => capybara_server_port if capybara_server_port + + def self.driven_by_selenium(config_name = "default", opts = {}) + preferences = opts.fetch(:preferences, {}).reverse_merge( + "intl.accept_languages" => "en" + ) + + options = { + :name => config_name + } + + if capybara_server_port + selenium_host = "http://selenium-#{config_name}:4444" + options = options.merge( + :url => selenium_host, + :browser => :remote + ) + end + + driven_by( + :selenium, + :using => Settings.system_test_headless ? :headless_firefox : :firefox, + :options => options + ) do |options| + preferences.each do |name, value| + options.add_preference(name, value) + end + options.binary = Settings.system_test_firefox_binary if Settings.system_test_firefox_binary + end end + driven_by_selenium + def before_setup super osm_website_app = create(:oauth_application, :name => "OpenStreetMap Web Site", :scopes => "write_api write_notes") diff --git a/test/system/embed_test.rb b/test/system/embed_test.rb index 2c798832be1..e906866ea35 100644 --- a/test/system/embed_test.rb +++ b/test/system/embed_test.rb @@ -10,9 +10,12 @@ class EmbedTest < ApplicationSystemTestCase end class GermanEmbedTest < ApplicationSystemTestCase - driven_by :selenium, :using => :headless_firefox, :options => { :name => :selenium_de } do |options| - options.add_preference("intl.accept_languages", "de") - end + driven_by_selenium( + "de", + :preferences => { + "intl.accept_languages" => "de" + } + ) test "shows localized report link" do visit export_embed_path @@ -21,9 +24,12 @@ class GermanEmbedTest < ApplicationSystemTestCase end class UnknownLanguageEmbedTest < ApplicationSystemTestCase - driven_by :selenium, :using => :headless_firefox, :options => { :name => :selenium_unknown_language } do |options| - options.add_preference("intl.accept_languages", "unknown-language") - end + driven_by_selenium( + "nolang", + :preferences => { + "intl.accept_languages" => "unknown-language" + } + ) test "shows report link in fallback language" do visit export_embed_path diff --git a/test/test_helper.rb b/test/test_helper.rb index ef159b0d475..1dc91f16266 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -36,7 +36,7 @@ def self.branch_coverage? require "webmock/minitest" require "minitest/focus" unless ENV["CI"] -WebMock.disable_net_connect!(:allow_localhost => true) +WebMock.disable_net_connect!(:allow_localhost => true, :allow => %w[selenium-default selenium-de selenium-nolang rails-app]) module ActiveSupport class TestCase @@ -44,15 +44,21 @@ class TestCase include ActiveJob::TestHelper include LibXML - # Run tests in parallel with specified workers - parallelize(:workers => :number_of_processors) + if ENV.key?("CAPYBARA_SERVER_PORT") + # Running in the devcontainer. Can't figure out how + # to run things in parallel at the moment, so for now + # we are not doing it. + else + # Run tests in parallel with specified workers + parallelize(:workers => :number_of_processors) - parallelize_setup do |worker| - SimpleCov.command_name "#{SimpleCov.command_name}-#{worker}" - end + parallelize_setup do |worker| + SimpleCov.command_name "#{SimpleCov.command_name}-#{worker}" + end - parallelize_teardown do - SimpleCov.result + parallelize_teardown do + SimpleCov.result + end end ##