Conformance test suite

.github/workflows/ci.yml

@@ -30,11 +30,11 @@ jobs:
         include:
           - target:
               os: linux
-            builder: ubuntu-18.04
+            builder: ubuntu-20.04
             shell: bash
           - target:
               os: macos
-            builder: macos-10.15
+            builder: macos-12
             shell: bash
           - target:
               os: windows
@@ -96,7 +96,7 @@ jobs:
       - name: Restore Nim DLLs dependencies (Windows) from cache
         if: runner.os == 'Windows'
         id: windows-dlls-cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: external/dlls-${{ matrix.target.cpu }}
           key: 'dlls-${{ matrix.target.cpu }}'

.github/workflows/conformance.yml

@@ -0,0 +1,101 @@
+name: Conformance Test
+
+on:
+  workflow_dispatch:
+    types: [run]
+  pull_request:
+
+concurrency: # Cancel stale PR builds (but not push builds)
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - os: linux
+            cpu: amd64
+          - os: linux
+            cpu: i386
+
+        branch: [version-1-4, version-1-6, devel]
+        include:
+          - target:
+              os: linux
+            builder: ubuntu-20.04
+            shell: bash
+
+    defaults:
+      run:
+        shell: ${{ matrix.shell }}
+
+    name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})'
+    runs-on: ${{ matrix.builder }}
+    continue-on-error: ${{ matrix.branch == 'devel' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install build dependencies (Linux i386)
+        if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
+        run: |
+          sudo dpkg --add-architecture i386
+          sudo apt-fast update -qq
+          sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
+            --no-install-recommends -yq gcc-multilib g++-multilib \
+            libssl-dev:i386
+          mkdir -p external/bin
+          cat << EOF > external/bin/gcc
+          #!/bin/bash
+          exec $(which gcc) -m32 "\$@"
+          EOF
+          cat << EOF > external/bin/g++
+          #!/bin/bash
+          exec $(which g++) -m32 "\$@"
+          EOF
+          chmod 755 external/bin/gcc external/bin/g++
+          echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
+
+      - name: Derive environment variables
+        run: |
+          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
+            PLATFORM=x64
+          else
+            PLATFORM=x86
+          fi
+          echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
+
+          ncpu=
+          MAKE_CMD="make"
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+          echo "ncpu=$ncpu" >> $GITHUB_ENV
+          echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
+
+      - name: Install CMake
+        run: |
+          if [ "${{ runner.os }}" == "Linux" ]; then
+            sudo apt-get install cmake
+          fi
+
+      - name: Build Nim and Nimble
+        run: |
+          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+          env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ matrix.branch }} \
+            QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \
+            bash build_nim.sh nim csources dist/nimble NimBinaries
+          echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
+
+      - name: Run tests
+        run: |
+          nim --version
+          nimble --version
+          nimble install -y --depsOnly
+          env NIMLANG="c" nimble conformance_test
+          env NIMLANG="cpp" nimble conformance_test

protobuf_serialization.nimble

@@ -50,3 +50,20 @@ task test, "Run all tests":
       echo "  \x1B[0;31m[FAILED]\x1B[0;37m ", path.split(DirSep)[^1]
       exec "exit 1"
 
+task conformance_test, "Run conformance tests":
+  let
+    pwd = thisDir()
+    conformance = pwd / "conformance"
+    test = pwd / "tests" / "conformance"
+
+  if not system.dirExists(conformance):
+    exec "git clone -b v22.0 --recurse-submodules https://github.com/protocolbuffers/protobuf/ " & conformance
+
+  withDir conformance:
+    exec "cmake . -Dprotobuf_BUILD_CONFORMANCE=ON"
+    exec "make conformance_test_runner"
+
+  exec "cp " & conformance / "conformance_test_runner" & " " & test
+  withDir test:
+    exec "nim c -d:ConformanceTest conformance_nim.nim"
+    exec "./conformance_test_runner --failure_list failure_list.txt conformance_nim"

protobuf_serialization/codec.nim

@@ -31,6 +31,8 @@ type
     # EndGroup = 4 # Not used
     Fixed32 = 5
 
+  SomePBInt* = int32 | int64 | uint32 | uint64
+
   FieldHeader* = distinct uint32
 
   # Scalar types used in `.proto` files
@@ -54,19 +56,20 @@ type
   sfixed64* = distinct int64 ## fixed-width signed integer
 
   pbool* = distinct bool
+  penum* = distinct int32
 
   pstring* = distinct string ## UTF-8-encoded string
   pbytes* = distinct seq[byte] ## byte sequence
 
   SomeScalar* =
-    pint32 | pint64 | puint32 | puint64 | sint32 | sint64 | pbool |
+    pint32 | pint64 | puint32 | puint64 | sint32 | sint64 | pbool | penum |
     fixed64 | sfixed64 | pdouble |
     pstring | pbytes |
     fixed32 | sfixed32 | pfloat
 
   # Mappings of proto type to wire type
   SomeVarint* =
-    pint32 | pint64 | puint32 | puint64 | sint32 | sint64 | pbool
+    pint32 | pint64 | puint32 | puint64 | sint32 | sint64 | pbool | penum
   SomeFixed64* = fixed64 | sfixed64 | pdouble
   SomeLengthDelim* = pstring | pbytes # Also messages and packed repeated fields
   SomeFixed32* = fixed32 | sfixed32 | pfloat
@@ -116,6 +119,7 @@ func toUleb(x: sint32): uint32 =
 template toUleb(x: pint64): uint64 = cast[uint64](x)
 template toUleb(x: pint32): uint32 = cast[uint32](x)
 template toUleb(x: pbool): uint8 = cast[uint8](x)
+template toUleb(x: penum): uint64 = cast[uint32](x)
 
 template fromUleb(x: uint64, T: type puint64): T = puint64(x)
 template fromUleb(x: uint64, T: type pbool): T = pbool(x != 0)
@@ -130,6 +134,7 @@ template fromUleb(x: uint64, T: type sint32): T =
 
 template fromUleb(x: uint64, T: type pint64): T = cast[T](x)
 template fromUleb(x: uint64, T: type pint32): T = cast[T](x)
+template fromUleb(x: uint64, T: type penum): T = cast[T](x)
 
 template toBytes*(x: SomeVarint): openArray[byte] =
   toBytes(toUleb(x), Leb128).toOpenArray()

protobuf_serialization/files/decldef.nim

@@ -26,7 +26,7 @@ type
   ReservedType* = enum
     String, Number, Range
   ProtoType* = enum
-    Field, Enum, EnumVal, ReservedBlock, Reserved, Message, File, Imported, Oneof, Package, ProtoDef
+    Field, Enum, EnumVal, ReservedBlock, Reserved, Message, File, Imported, Oneof, Package, ProtoDef, Extend
   Presence* = enum
     Singular, Repeated, Optional, Required
   ProtoNode* = ref object
@@ -62,6 +62,9 @@ type
       definedEnums*: seq[ProtoNode]
       fields*: seq[ProtoNode]
       nested*: seq[ProtoNode]
+    of Extend:
+      extending*: string
+      extendedFields*: seq[ProtoNode]
     of Package:
       packageName*: string
       messages*: seq[ProtoNode]
@@ -148,6 +151,14 @@ proc `$`*(node: ProtoNode): string =
           messages &= $message & "\n"
         data &= messages[0..^2].indent(1, "  ")
       result &= data.indent(1, "  ")
+    of Extend:
+      result = "Extension of $1 with fields:".format(node.extending)
+      var data = ""
+      var fields = "\n"
+      for field in node.extendedFields:
+        fields &= $field & "\n"
+      data &= fields[0..^2].indent(1, "  ")
+      result &= data.indent(1, "  ")
     of File:
       result = "Protobuf file with syntax $1\n".format(
         node.syntax)