azure-pipelines.yml

# Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
#
# Azure Pipelines CI build configuration
# Documentation at https://aka.ms/yaml

variables:
  #
  # If updating VERILATOR_VERSION, OPENOCD_VERSION or TOOLCHAIN_VERSION, update
  # the definitions in util/container/Dockerfile as well.
  #
  VERILATOR_VERSION: 4.104
  OPENOCD_VERSION: 0.11.0
  TOOLCHAIN_PATH: /opt/buildcache/riscv
  VERIBLE_VERSION: v0.0-808-g1e17daa
  # Release tag from https://github.com/lowRISC/lowrisc-toolchains/releases
  TOOLCHAIN_VERSION: 20210412-1
  # This controls where builds happen, and gets picked up by build_consts.sh.
  BUILD_ROOT: $(Build.ArtifactStagingDirectory)
  VIVADO_VERSION: "2020.1"

trigger:
  batch: true
  branches:
    include:
    - "*"
  tags:
    include:
    - "*"
pr:
  branches:
    include:
    - "*"

jobs:
- job: lint
  displayName: Run code quality checks (quick lint)
  pool:
    vmImage: ubuntu-18.04
  steps:
  - bash: |
      sudo apt-get remove -y clang-6.0 libclang-common-6.0-dev libclang1-6.0 libllvm6.0
    displayName: Remove existing Clang installation
  - template: ci/install-package-dependencies.yml
    ## !!!
    ##
    ##   The steps below here are duplicated in ci/jobs/quick-lint.sh
    ##   to allow developers to "run CI" locally. Keep them in sync.
    ##
    ## !!!
  - bash: ci/scripts/show-env.sh
    displayName: Display environment information
  - bash: ci/scripts/lint-commits.sh $SYSTEM_PULLREQUEST_TARGETBRANCH
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: Check commit metadata
  - bash: ci/scripts/check-licence-headers.sh $SYSTEM_PULLREQUEST_TARGETBRANCH
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: Check Licence Headers
  - bash: ci/scripts/exec-check.sh
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: Check executable bits
  - bash: ci/scripts/python-lint.sh $SYSTEM_PULLREQUEST_TARGETBRANCH
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: Run Python lint (flake8)
    continueOnError: true
  - bash: ci/scripts/mypy.sh
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: Run Python lint (mypy)
  - bash: ci/scripts/check-generated.sh
    displayName: Ensure all generated files are clean and up-to-date
  - bash: ci/scripts/clang-format.sh $SYSTEM_PULLREQUEST_TARGETBRANCH
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: Use clang-format to check C/C++ coding style
  - bash: ci/scripts/include-guard.sh $SYSTEM_PULLREQUEST_TARGETBRANCH
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: Check formatting on header guards
  - bash: ci/scripts/verible-lint.sh rtl
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: Style-Lint RTL Verilog source files with Verible
  - bash: ci/scripts/verible-lint.sh dv
    condition: eq(variables['Build.Reason'], 'PullRequest')
    displayName: Style-Lint DV Verilog source files with Verible
  - bash: ci/scripts/build-docs.sh
    displayName: Render documentation
  - bash: ci/scripts/build-site.sh
    displayName: Render landing site
  - bash: ci/scripts/get-build-type.sh "$SYSTEM_PULLREQUEST_TARGETBRANCH" "$(Build.Reason)"
    displayName: Check what kinds of changes the PR contains
    name: DetermineBuildType

- job: slow_lints
  displayName: Run code quality checks (in-depth lint)
  dependsOn: lint
  pool:
    vmImage: ubuntu-18.04
  steps:
  - template: ci/install-package-dependencies.yml
  - bash: ci/scripts/check-vendoring.sh
    displayName: Check vendored directories are up-to-date

- job: sw_build
  displayName: Build Software for Earl Grey toplevel design
  dependsOn: lint
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'))
  pool:
    vmImage: ubuntu-18.04
  steps:
  - template: ci/install-package-dependencies.yml
  - bash: |
      set -x
      sudo util/get-toolchain.py \
        --install-dir="$TOOLCHAIN_PATH" \
        --release-version="$TOOLCHAIN_VERSION" \
        --update
    displayName: Install toolchain
  - bash: |
      . util/build_consts.sh
      ./meson_init.sh -A
      ninja -C "$OBJ_DIR" all
    displayName: Build embedded targets
  - bash: |
      . util/build_consts.sh
      ninja -C "$OBJ_DIR" test
    displayName: Run unit tests
  - bash: |
      . util/build_consts.sh
      # Remove all Nexys Video-related build artifacts, which will be produced
      # by the sw_build_nexysvideo job below (see comment there).
      find "$BIN_DIR/sw/device" -name '*fpga_nexysvideo*' -type f -delete
    displayName: Delete all Nexys Video build artifacts
  - template: ci/upload-artifacts-template.yml
    parameters:
      includePatterns:
        - "/sw/***"

# Software targeting the Nexys Video board is produced by patching the source
# tree before building. This produces a full sw build tree, however only the
# artifacts with "nexysvideo" in the name are actually the ones we're looking
# for. Everything else will be discarded.
# TODO: This is a rather ugly hack, which will go away once we properly support
# building more than one top-level design with different parametrizations.
# Work towards this goal is tracked in issue #4669.
- job: sw_build_nexysvideo
  displayName: Build Software for Earl Grey toplevel design targeting the Nexys Video board
  dependsOn: lint
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'))
  pool:
    vmImage: ubuntu-18.04
  steps:
  - template: ci/install-package-dependencies.yml
  - bash: |
      set -x
      sudo util/get-toolchain.py \
        --install-dir="$TOOLCHAIN_PATH" \
        --release-version="$TOOLCHAIN_VERSION" \
        --update
    displayName: Install toolchain
  - bash: |
      . util/build_consts.sh
      ./hw/top_earlgrey/util/top_earlgrey_reduce.py
      ./meson_init.sh -A
      ninja -C "$OBJ_DIR" all

      # Delete all build artifacts which are *not* for the Nexys Video board.
      find "$BIN_DIR/sw/device" -not -name '*fpga_nexysvideo*' -type f -delete
    displayName: Build embedded targets
  - bash: |
      . util/build_consts.sh
      ninja -C "$OBJ_DIR" test
    displayName: Run unit tests
  - template: ci/upload-artifacts-template.yml
    parameters:
      includePatterns:
        - "/sw/device/**/*nexysvideo*"

# We continue building with GCC, despite defaulting to Clang. This is a copy of
# `sw_build` with `meson_init.sh` configured with the GCC toolchain, instead of
# the default toolchain.
- job: sw_build_gcc
  displayName: Build Software for Earl Grey toplevel design (with GCC)
  dependsOn: lint
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'))
  pool:
    vmImage: ubuntu-18.04
  steps:
  - template: ci/install-package-dependencies.yml
  - bash: |
      set -x
      sudo util/get-toolchain.py \
        --install-dir="$TOOLCHAIN_PATH" \
        --release-version="$TOOLCHAIN_VERSION" \
        --update
    displayName: Install toolchain
  - bash: |
      . util/build_consts.sh
      ./meson_init.sh -A \
        -t "$TOOLCHAIN_PATH/meson-riscv32-unknown-elf-gcc.txt"
      ninja -C "$OBJ_DIR" all
    displayName: Build embedded targets
  - bash: |
      . util/build_consts.sh
      ninja -C "$OBJ_DIR" test
    displayName: Run unit tests

- job: chip_earlgrey_verilator
  displayName: Build Verilator simulation of the Earl Grey toplevel design
  dependsOn: lint
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'))
  pool:
    vmImage: ubuntu-18.04
  steps:
  - template: ci/install-package-dependencies.yml
  - bash: |
      python3 --version
      fusesoc --version
      verilator --version
      verible-verilog-lint --version
    displayName: Display environment
  - bash: |
      . util/build_consts.sh
      mkdir -p "$OBJ_DIR/hw"
      mkdir -p "$BIN_DIR/hw/top_earlgrey"

      # Compile the simulation without threading; the runners provided by
      # Azure provide two virtual CPUs, which seems to equal one physical
      # CPU (at most); the use of threading slows down the simulation.
      fusesoc --cores-root=. \
        run --flag=fileset_top --target=sim --setup --build \
        --build-root="$OBJ_DIR/hw" \
        lowrisc:systems:chip_earlgrey_verilator \
        --verilator_options="--no-threads"

      cp "$OBJ_DIR/hw/sim-verilator/Vchip_earlgrey_verilator" \
        "$BIN_DIR/hw/top_earlgrey"
    displayName: Build simulation with Verilator
  - template: ci/upload-artifacts-template.yml
    parameters:
      includePatterns:
        - "/hw/top_earlgrey/Vchip_earlgrey_verilator"

- job: chip_englishbreakfast_verilator
  displayName: Build Verilator simulation of the English Breakfast toplevel design
  dependsOn: lint
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'))
  pool:
    vmImage: ubuntu-18.04
  steps:
  - template: ci/install-package-dependencies.yml
  - bash: |
      python3 --version
      fusesoc --version
      verilator --version
      verible-verilog-lint --version
    displayName: Display environment
  - bash: |
      . util/build_consts.sh
      mkdir -p "$OBJ_DIR/hw"
      mkdir -p "$BIN_DIR/hw/top_englishbreakfast"

      # Compile the simulation without threading; the runners provided by
      # Azure provide two virtual CPUs, which seems to equal one physical
      # CPU (at most); the use of threading slows down the simulation.
      fusesoc --cores-root=. \
        run --flag=fileset_topgen --target=sim --setup --build \
        --build-root="$OBJ_DIR/hw" \
        lowrisc:systems:chip_englishbreakfast_verilator \
        --verilator_options="--no-threads"

      cp "$OBJ_DIR/hw/sim-verilator/Vchip_englishbreakfast_verilator" \
        "$BIN_DIR/hw/top_englishbreakfast"
    displayName: Build simulation with Verilator
  - template: ci/upload-artifacts-template.yml
    parameters:
      includePatterns:
        - "/hw/top_englishbreakfast/Vchip_englishbreakfast_verilator"

- job: execute_verilated_tests
  displayName: Execute tests on the Verilated system
  pool:
    vmImage: ubuntu-18.04
  dependsOn:
    - chip_earlgrey_verilator
    - sw_build
  steps:
  - template: ci/install-package-dependencies.yml
  - template: ci/download-artifacts-template.yml
    parameters:
      downloadPartialBuildBinFrom:
        - chip_earlgrey_verilator
        - sw_build
  - bash: |
      # Install an additional pytest dependency for result upload.
      pip3 install pytest-azurepipelines

      . util/build_consts.sh
      pytest --version
      pytest test/systemtest/earlgrey/test_sim_verilator.py \
        --log-cli-level=DEBUG \
        --test-run-title="Run system tests with Verilator simulation" \
        --napoleon-docstrings
    displayName: Execute tests

- template: ci/run-riscv-compliance.yml
  parameters:
    rvc_test_suites:
      - rv32i

- template: ci/run-riscv-compliance.yml
  parameters:
    rvc_test_suites:
      - rv32im
      - rv32imc
      - rv32Zicsr

- job: otbn_standalone_tests
  displayName: Run OTBN Smoke Test
  dependsOn: lint
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.hasOTBNChanges'], '1'))
  pool:
    vmImage: ubuntu-18.04
  timeoutInMinutes: 10
  steps:
  - template: ci/install-package-dependencies.yml
  - bash: |
      set -x
      sudo util/get-toolchain.py \
        --install-dir="$TOOLCHAIN_PATH" \
        --release-version="$TOOLCHAIN_VERSION" \
        --update
      echo "##vso[task.prependpath]$TOOLCHAIN_PATH/bin"
    displayName: Install toolchain
  - bash: |
      python3 --version
      fusesoc --version
      verilator --version
    displayName: Display environment
  - bash: |
      make -C hw/ip/otbn/dv/otbnsim test
    displayName: OTBN ISS Test
  - bash: |
      ./hw/ip/otbn/dv/smoke/run_smoke.sh
    displayName: OTBN Smoke Test
  - bash: |
      make -C hw/ip/otbn/util asm-check
    displayName: Assemble and link code snippets

- job: chip_earlgrey_nexysvideo
  displayName: Build NexysVideo variant of the Earl Grey toplevel design using Vivado
  dependsOn:
    - lint
    # The bootrom is built into the FPGA image at synthesis time.
    - sw_build_nexysvideo
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyDvChanges'], '0'))
  pool: ci-public
  timeoutInMinutes: 120 # 2 hours
  steps:
  - template: ci/install-package-dependencies.yml
  - template: ci/download-artifacts-template.yml
    parameters:
      downloadPartialBuildBinFrom:
        - sw_build_nexysvideo
  - bash: |
      set -e
      . util/build_consts.sh

      module load "xilinx/vivado/$(VIVADO_VERSION)"

      mkdir -p "$OBJ_DIR/hw"
      mkdir -p "$BIN_DIR/hw/top_earlgrey"

      ./hw/top_earlgrey/util/top_earlgrey_reduce.py

      BOOTROM_VMEM="$BIN_DIR/sw/device/boot_rom/boot_rom_fpga_nexysvideo.scr.40.vmem"
      test -f "$BOOTROM_VMEM"
      OTP_VMEM="$BIN_DIR/sw/device/otp_img/otp_img_fpga_nexysvideo.vmem"
      test -f "$OTP_VMEM"

      fusesoc --cores-root=. \
        run --flag=fileset_top --target=synth --setup --build \
        --build-root="$OBJ_DIR/hw" \
        lowrisc:systems:chip_earlgrey_nexysvideo \
        --BootRomInitFile="$BOOTROM_VMEM" \
        --OtpCtrlMemInitFile="$OTP_VMEM"

      cp "$OBJ_DIR/hw/synth-vivado/lowrisc_systems_chip_earlgrey_nexysvideo_0.1.bit" \
        "$BIN_DIR/hw/top_earlgrey"
    displayName: Build bitstream with Vivado
  - bash: |
      . util/build_consts.sh
      echo Synthesis log
      cat $OBJ_DIR/hw/synth-vivado/lowrisc_systems_chip_earlgrey_nexysvideo_0.1.runs/synth_1/runme.log || true

      echo Implementation log
      cat $OBJ_DIR/hw/synth-vivado/lowrisc_systems_chip_earlgrey_nexysvideo_0.1.runs/impl_1/runme.log || true
    condition: always()
    displayName: Display synthesis and implementation logs
  - template: ci/upload-artifacts-template.yml
    parameters:
      includePatterns:
        - "/hw/top_earlgrey/lowrisc_systems_chip_earlgrey_nexysvideo_0.1.bit"

- job: chip_englishbreakfast_cw305
  displayName: Build CW305 variant of the English Breakfast toplevel design using Vivado
  dependsOn:
    - lint
    # The bootrom is built into the FPGA image at synthesis time.
    # Currently, we can't have different versions of binaries in $BIN_DIR. Consequently, we are
    # using the NexysVideo bootrom here and the resulting CW305 bitstream is not functional.
    # By generating the CW305 bootrom binary we would break execute_fpga_tests executed on the
    # NexysVideo.
    - sw_build_nexysvideo
  condition: and(succeeded(), eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyDvChanges'], '0'))
  pool: ci-public
  timeoutInMinutes: 120 # 2 hours
  steps:
  - template: ci/install-package-dependencies.yml
  - template: ci/download-artifacts-template.yml
    parameters:
      downloadPartialBuildBinFrom:
        - sw_build_nexysvideo
  - bash: |
      set -e
      . util/build_consts.sh

      module load "xilinx/vivado/$(VIVADO_VERSION)"

      mkdir -p "$OBJ_DIR/hw"
      mkdir -p "$BIN_DIR/hw/top_englishbreakfast"

      BOOTROM_VMEM="$BIN_DIR/sw/device/boot_rom/boot_rom_fpga_nexysvideo.scr.40.vmem"
      test -f "$BOOTROM_VMEM"

      fusesoc --cores-root=. \
        run --flag=fileset_topgen --target=synth --setup --build \
        --build-root="$OBJ_DIR/hw" \
        lowrisc:systems:chip_englishbreakfast_cw305 \
        --BootRomInitFile="$BOOTROM_VMEM"

      cp "$OBJ_DIR/hw/synth-vivado/lowrisc_systems_chip_englishbreakfast_cw305_0.1.bit" \
        "$BIN_DIR/hw/top_englishbreakfast"
    displayName: Build bitstream with Vivado
  - template: ci/upload-artifacts-template.yml
    parameters:
      includePatterns:
        - "/hw/top_englishbreakfast/lowrisc_systems_chip_englishbreakfast_cw305_0.1.bit"

- job: execute_fpga_tests
  displayName: Execute tests on FPGA
  pool: FPGA
  timeoutInMinutes: 30
  dependsOn:
    - chip_earlgrey_nexysvideo
    - sw_build_nexysvideo
    - sw_build
  steps:
  - template: ci/install-package-dependencies.yml
  - template: ci/download-artifacts-template.yml
    parameters:
      downloadPartialBuildBinFrom:
        - chip_earlgrey_nexysvideo
        - sw_build_nexysvideo
        - sw_build
  - bash: |
      set -e

      module load "xilinx/vivado/$(VIVADO_VERSION)"

      # Install an additional pytest dependency for result upload.
      pip3 install pytest-azurepipelines

      . util/build_consts.sh
      pytest --version
      pytest test/systemtest/earlgrey/test_fpga_nexysvideo.py \
        --log-cli-level=DEBUG \
        --test-run-title="Run system tests on Nexys Video FPGA board" \
        --napoleon-docstrings
    displayName: Execute tests

- job: deploy_release_artifacts
  displayName: Package and deploy release distribution
  pool:
    vmImage: ubuntu-18.04
  dependsOn:
    - lint
    - sw_build
    - chip_earlgrey_verilator
    - chip_earlgrey_nexysvideo
  condition: and(eq(dependencies.lint.outputs['DetermineBuildType.onlyDocChanges'], '0'), eq(dependencies.lint.outputs['DetermineBuildType.onlyDvChanges'], '0'))
  steps:
  - template: ci/install-package-dependencies.yml
  - template: ci/download-artifacts-template.yml
    parameters:
      downloadPartialBuildBinFrom:
        - sw_build
        - sw_build_nexysvideo
        - chip_earlgrey_verilator
        - chip_earlgrey_nexysvideo
        - chip_englishbreakfast_verilator
  - bash: |
      . util/build_consts.sh

      util/make_distribution.sh

      tar --list -f $BIN_DIR/opentitan-*.tar.xz
      # Put the resulting tar file into a directory the |publish| step below can reference.
      mkdir "$BUILD_ROOT/dist-final"
      mv $BIN_DIR/opentitan-*.tar.xz "$BUILD_ROOT/dist-final"
    displayName: Create final dist directory out of partial ones
  - publish: $(Build.ArtifactStagingDirectory)/dist-final
    artifact: opentitan-dist
    displayName: Upload release artifacts as Azure artifact
  - task: GithubRelease@0
    displayName: Upload to GitHub releases (only tags)
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
    inputs:
      gitHubConnection: opentitan-release-upload
      repositoryName: lowrisc/opentitan
      addChangeLog: false
      assets: |
          $(Build.ArtifactStagingDirectory)/dist-final/*


- job: build_docker_containers
  displayName: "Build Docker Containers"
  pool:
    vmImage: ubuntu-18.04
  dependsOn:
    - lint
  steps:
  - task: Docker@2
    displayName: Build Developer Utility Container
    inputs:
      command: build
      Dockerfile: ./util/container/Dockerfile
      buildContext: .
  - task: Docker@2
    displayName: Build Documentation Builder Container
    inputs:
      command: build
      tags: gcr.io/active-premise-257318/builder
      Dockerfile: ./site/docs/builder.Dockerfile
      buildContext: .
  - task: Docker@2
    displayName: Build Documentation Redirector Container
    inputs:
      command: build
      Dockerfile: ./site/redirector/Dockerfile
      buildContext: ./site/redirector