Dockerfile

# This Dockerfile builds OpenMS, the TOPP tools, pyOpenMS and thidparty tools.
# It also adds a basic streamlit server that serves a pyOpenMS-based app.
# hints:
# build image and give it a name (here: streamlitapp) with: docker build --no-cache -t streamlitapp:latest --secret id=github_token,env=GITHUB_TOKEN . 2>&1 | tee build.log
# check if image was build: docker image ls
# run container: docker run -p 8501:8501 streamlitappsimple:latest
# debug container after build (comment out ENTRYPOINT) and run container with interactive /bin/bash shell
# prune unused images/etc. to free disc space (e.g. might be needed on gitpod). Use with care.: docker system prune --all --force
#
# OpenMS version: OPENMS_BRANCH below takes a release branch (release/3.5.0),
# a release tag (v3.6.0) or a branch such as develop. Override it per build with
# --build-arg OPENMS_BRANCH=v3.6.0, or change the default in an app's copy.
# The same steps build every OpenMS from 3.5 on: dependencies come from Ubuntu
# packages plus Apache Arrow/Parquet (required from 3.6 on), never from the
# contrib or vcpkg submodules, and the pyOpenMS wheel is built the way the
# checked-out source defines it (autowrap/Cython up to 3.5, nanobind from 3.6).
# The same file builds amd64 and arm64.

FROM ubuntu:24.04 AS setup-build-system
# FDmerge: the OpenMS branch with FLASHAnnotator (same as the desktop build).
ARG OPENMS_REPO=https://github.com/t0mdavid-m/OpenMS.git
ARG OPENMS_BRANCH=FDmerge
ARG PORT=8501
# GitHub repository whose latest release supplies the bundled Windows
# installer. Empty: `github-user` and `repository-name` from settings.json,
# so every app downloads its own installer without editing this file.
ARG GITHUB_USER=
ARG GITHUB_REPO=
# Python of the app's environment. pyOpenMS 3.6 needs 3.11 or newer.
ARG PYTHON_VERSION=3.11
# Apache Arrow/Parquet major version. OpenMS 3.6 needs 23 or newer; it is pinned
# so the libraries do not move under an image between two builds.
ARG ARROW_VERSION=23

USER root

# Keep apt from stopping at interactive prompts: tzdata, pulled in by
# later packages, otherwise waits for a timezone and the build hangs.
ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC

# Install required Ubuntu packages. cmake is Ubuntu 24.04's 3.28 (OpenMS 3.5
# needs 3.24, pyOpenMS 3.6 3.26).
RUN apt-get -y update
RUN apt-get install -y --no-install-recommends --no-install-suggests g++ autoconf automake patch libtool make cmake pkg-config git gpg wget ca-certificates curl jq libgtk2.0-dev openjdk-8-jdk cron
RUN update-ca-certificates
RUN apt-get install -y --no-install-recommends --no-install-suggests libsvm-dev libeigen3-dev coinor-libcbc-dev coinor-libcoinmp-dev libglpk-dev libzip-dev zlib1g-dev libzstd-dev libxerces-c-dev libbz2-dev libcurl4-openssl-dev libomp-dev libhdf5-dev
RUN apt-get install -y --no-install-recommends --no-install-suggests libboost-date-time-dev \
                                                                     libboost-iostreams-dev \
                                                                     libboost-regex-dev \
                                                                     libboost-math-dev \
                                                                     libboost-random-dev
RUN apt-get install -y --no-install-recommends --no-install-suggests qt6-base-dev libqt6svg6-dev libqt6opengl6-dev libqt6openglwidgets6 libgl-dev

# Apache Arrow/Parquet from Apache's apt repository, as OpenMS's own image does.
# Appending Apache's KEYS keeps the repository usable across key rotations.
# downloads.apache.org sometimes does not answer GitHub's arm64 runners (wget
# hung until it gave up, exit code 4), so wget gets a timeout and retries and
# falls back to dist.apache.org, the release tree it mirrors.
RUN . /etc/os-release \
    && wget -q https://packages.apache.org/artifactory/arrow/ubuntu/apache-arrow-apt-source-latest-${VERSION_CODENAME}.deb \
    && apt-get install -y --no-install-recommends ./apache-arrow-apt-source-latest-${VERSION_CODENAME}.deb \
    && rm apache-arrow-apt-source-latest-${VERSION_CODENAME}.deb \
    && (wget -q --timeout=30 --tries=3 -O /tmp/arrow-KEYS https://downloads.apache.org/arrow/KEYS \
        || wget -q --timeout=30 --tries=3 -O /tmp/arrow-KEYS https://dist.apache.org/repos/dist/release/arrow/KEYS) \
    && cat /tmp/arrow-KEYS >> /usr/share/keyrings/apache-arrow-apt-source.asc \
    && rm /tmp/arrow-KEYS \
    && printf 'Package: libarrow* libparquet*\nPin: version %s.*\nPin-Priority: 999\n' "${ARROW_VERSION}" > /etc/apt/preferences.d/arrow \
    && apt-get update \
    && apt-get install -y --no-install-recommends --no-install-suggests libarrow-dev libparquet-dev

# Install Github CLI
RUN (type -p wget >/dev/null || (apt-get update && apt-get install wget -y)) \
	&& mkdir -p -m 755 /etc/apt/keyrings \
	&& wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
	&& chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
	&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
	&& apt-get update \
	&& apt-get install gh -y

# Download and install miniforge (x86_64 or aarch64, whichever this build runs on).
ENV PATH="/root/miniforge3/bin:${PATH}"
RUN wget -q \
    https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh \
    && bash Miniforge3-Linux-$(uname -m).sh -b \
    && rm -f Miniforge3-Linux-$(uname -m).sh
RUN mamba --version

# Make /root traversable so the entrypoint can `source
# /root/miniforge3/bin/activate ...` when the container runs as a non-root
# user (apptainer/singularity maps the host UID into the container; the
# default ubuntu /root is 0700 which would block path traversal). +x only,
# not +r, so the directory listing remains private.
RUN chmod o+x /root

# Setup mamba environment.
RUN mamba create -n streamlit-env python=${PYTHON_VERSION}
RUN echo "mamba activate streamlit-env" >> ~/.bashrc
SHELL ["/bin/bash", "--rcfile", "~/.bashrc"]
SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"]

# Clone the OpenMS branch or tag, and of its submodules only THIRDPARTY: the
# dependencies come from the packages above, not from contrib or vcpkg.
RUN git clone --depth=1 -b ${OPENMS_BRANCH} --single-branch ${OPENMS_REPO} /OpenMS
WORKDIR /OpenMS

# Python packages for the pyOpenMS build, as the checked-out source lists them.
# OpenMS 3.6 builds pyOpenMS with nanobind (build requirements in
# src/pyOpenMS/pyproject.toml); earlier versions with autowrap and Cython
# (src/pyOpenMS/requirements_bld.txt). src/CMakeLists.txt defines the
# pyopenms_wheel target only in the nanobind generation.
# cython<3.3: Cython 3.3.0 (2026-08-22) removed Cython.Compiler.Nodes.CConstOrVolatileTypeNode,
# which autowrap up to 0.24 needs, breaking `make pyopenms`.
# FLASHApp: for FDmerge (requirements_bld.txt) autowrap is pinned to 0.23.0: its
# create_cpp_extension.py passes include_boost, which autowrap 0.24 removed.
RUN pip install --upgrade pip && python -m pip install -U setuptools wheel pandas numpy pytest \
    && if grep -q pyopenms_wheel src/CMakeLists.txt; then \
        python -c "import tomllib; print('\n'.join(tomllib.load(open('src/pyOpenMS/pyproject.toml', 'rb'))['build-system']['requires']))" > /tmp/pyopenms-build.txt; \
    elif [ -f src/pyOpenMS/requirements_bld.txt ]; then \
        { cat src/pyOpenMS/requirements_bld.txt; echo; printf '%s\n' nose "cython<3.3" "autowrap==0.23.0"; } > /tmp/pyopenms-build.txt; \
    else \
        printf '%s\n' nose "cython>=3.1.0,<3.3" "autowrap<=0.24" > /tmp/pyopenms-build.txt; \
    fi \
    && cat /tmp/pyopenms-build.txt \
    && python -m pip install -r /tmp/pyopenms-build.txt

# Pull Linux compatible third-party dependencies and store them in directory thirdparty.
RUN mkdir /thirdparty && \
    git submodule update --init --depth=1 THIRDPARTY && \
    cp -r THIRDPARTY/All/* /thirdparty && \
    if [ -d "THIRDPARTY/Linux/$(uname -m)" ]; then \
        cp -r THIRDPARTY/Linux/$(uname -m)/* /thirdparty; \
    fi && \
    rm -rf THIRDPARTY && \
    chmod -R +x /thirdparty
ENV PATH="/thirdparty/LuciPHOr2:/thirdparty/MSGFPlus:/thirdparty/Sirius:/thirdparty/ThermoRawFileParser:/thirdparty/Comet:/thirdparty/Fido:/thirdparty/MaRaCluster:/thirdparty/MyriMatch:/thirdparty/OMSSA:/thirdparty/Percolator:/thirdparty/SpectraST:/thirdparty/XTandem:/thirdparty/crux:/thirdparty/Sage:${PATH}"

# Configure OpenMS and pyOpenMS.
FROM setup-build-system AS configure-openms
WORKDIR /

# Configure under plain bash, with the system cmake, so that the C++
# dependencies resolve to the Ubuntu packages and never to libraries of the
# mamba environment (whose libstdc++ or yaml-cpp can differ from the system
# compiler's). pyOpenMS gets the environment's interpreter explicitly.
# Options a version does not know are ignored with a warning, so one command
# serves 3.5 and 3.6+: OPENMS_USE_VCPKG=OFF takes the system packages,
# WITH_UV=OFF builds against this Python instead of a uv-made venv, and
# WITH_THERMO_RAW=OFF leaves out the .NET Thermo reader (FileConverter converts
# Thermo RAW files with ThermoRawFileParser and mono, installed below).
SHELL ["/bin/bash", "-c"]
RUN cmake -S /OpenMS -B /openms-build \
        -DCMAKE_BUILD_TYPE='Release' \
        -DCMAKE_PREFIX_PATH='/usr/;/usr/local' \
        -DHAS_XSERVER=OFF \
        -DBOOST_USE_STATIC=OFF \
        -DOPENMP=ON \
        -DOPENMS_USE_VCPKG=OFF \
        -DWITH_UV=OFF \
        -DWITH_THERMO_RAW=OFF \
        -DPYOPENMS=ON \
        -DPY_MEMLEAK_DISABLE=On \
        -DPython_EXECUTABLE=/root/miniforge3/envs/streamlit-env/bin/python

# Build OpenMS and pyOpenMS.
FROM configure-openms AS compile-openms
WORKDIR /openms-build

# Build TOPP tools.
RUN make -j4 TOPP

# pyOpenMS runs in the mamba environment's Python, whose OpenSSL is newer than
# Ubuntu's. Loading libOpenMS (which links the system libcurl) next to it mixes
# the two libssl/libcrypto builds and the import fails ("OPENSSL_3.3.0 not
# found"); pyOpenMS 3.6 imports itself during the build to generate its stubs.
# Taking the environment's libraries first keeps one OpenSSL in the process;
# they are newer and keep the symbol versions the system libraries ask for.
ENV LD_LIBRARY_PATH="/root/miniforge3/envs/streamlit-env/lib"

# Build the pyOpenMS wheel (pyopenms_wheel from 3.6 on, pyopenms before).
RUN if grep -q pyopenms_wheel /OpenMS/src/CMakeLists.txt; then \
        make -j4 pyopenms_wheel && mkdir -p /pyopenms-wheel && cp pyopenms_wheels/*.whl /pyopenms-wheel/; \
    else \
        make -j4 pyopenms && mkdir -p /pyopenms-wheel && cp pyOpenMS/dist/*.whl /pyopenms-wheel/; \
    fi && ls /pyopenms-wheel

SHELL ["mamba", "run", "-n", "streamlit-env", "/bin/bash", "-c"]
RUN pip install /pyopenms-wheel/*.whl && rm -rf /pyopenms-wheel

# Install other dependencies (excluding pyopenms)
COPY requirements.txt ./requirements.txt
RUN grep -Ev '^pyopenms([=<>!~].*)?$' requirements.txt > requirements_cleaned.txt && mv requirements_cleaned.txt requirements.txt
RUN pip install -r requirements.txt

WORKDIR /
RUN mkdir openms

# Copy TOPP tools bin directory, add to PATH.
RUN cp -r openms-build/bin /openms/bin
ENV PATH="/openms/bin/:${PATH}"

# Copy TOPP tools bin directory, add to PATH.
RUN cp -r openms-build/lib /openms/lib
ENV LD_LIBRARY_PATH="/openms/lib/:/root/miniforge3/envs/streamlit-env/lib"

# Copy share folder, add to PATH, remove source directory.
RUN cp -r OpenMS/share/OpenMS /openms/share
RUN rm -rf OpenMS
ENV OPENMS_DATA_PATH="/openms/share/"

# Remove build directory.
RUN rm -rf openms-build

# Check that pyOpenMS loads and finds its data, and that the TOPP tools run.
RUN python -c "import pyopenms; print('pyOpenMS', pyopenms.__version__, pyopenms.EmpiricalFormula('C6H12O6').getMonoWeight())" \
    && test -x /openms/bin/FileInfo \
    && ! ldd /openms/bin/FileInfo | grep 'not found'

# Build openms-insight, the visualization components of the viewers (quick).
# Placed after the slow OpenMS build so that changes to openms-insight do not
# invalidate the OpenMS compile cache; it is installed in the run-app stage.
# TEMPORARY (develop only): built from openms-insight's development branch
# (main) and installed over the PyPI release from requirements.txt. Revert this
# before a FLASHApp release: releases use a published openms-insight version.
FROM node:21 AS insight-build

ARG INSIGHT_REPO=https://github.com/t0mdavid-m/openms-insight.git
ARG INSIGHT_REF=main

ADD https://api.github.com/repos/t0mdavid-m/openms-insight/git/refs/heads/$INSIGHT_REF version.json

RUN git clone -b ${INSIGHT_REF} --single-branch ${INSIGHT_REPO} /openms-insight
WORKDIR /openms-insight/js-component
RUN npm ci && npm run build
# The Python package ships the bundle from openms_insight/js-component/dist
RUN mkdir -p ../openms_insight/js-component && cp -r dist ../openms_insight/js-component/

# Prepare and run streamlit app.
FROM compile-openms AS run-app

# Install Redis server for job queue and nginx for load balancing.
# Redis data lives under $RUNTIME_DIR at runtime (see entrypoint.sh) so no
# /var/lib/redis setup is needed - that path is not writable under Apptainer.
RUN apt-get update && apt-get install -y --no-install-recommends redis-server nginx \
    && rm -rf /var/lib/apt/lists/*

# Mono runs the ThermoRawFileParser (THIRDPARTY/All, a .NET Framework build)
# that FileConverter calls to convert Thermo RAW files to mzML on Linux.
RUN apt-get update && apt-get install -y --no-install-recommends mono-complete \
    && rm -rf /var/lib/apt/lists/* \
    && mono --version

# Command line tools an app installs from conda-forge (SIRIUS, for example),
# one package spec per line in its optional conda-requirements.txt. Python
# packages stay in requirements.txt. The glob lets the file be absent; this
# step comes after the OpenMS build, so changing the list never rebuilds it.
COPY requirements.txt conda-requirements.tx[t] /tmp/app-env/
# A line ending in `# [x86_64]` or `# [aarch64]` is installed only on that
# machine (`uname -m`), for tools conda-forge builds for one architecture.
RUN touch /tmp/app-env/conda-requirements.txt \
    && awk -v arch="$(uname -m)" ' \
        /#[[:space:]]*\[[a-z0-9_]+\][[:space:]]*$/ { tag = $0; sub(/.*\[/, "", tag); sub(/\].*/, "", tag); if (tag != arch) next } \
        { sub(/#.*/, ""); if ($0 ~ /[^[:space:]]/) print }' \
        /tmp/app-env/conda-requirements.txt > /tmp/app-env/conda-specs.txt \
    && if [ -s /tmp/app-env/conda-specs.txt ]; then \
        mamba install -y -n streamlit-env -c conda-forge --file /tmp/app-env/conda-specs.txt; \
    fi && rm -rf /tmp/app-env

# Create Redis data directory. Default 0755 root-owned is enough: the docker
# entrypoint runs as root (can write regardless of mode), and the apptainer
# entrypoint relocates Redis state to /tmp/openms-runtime-* so this dir is
# never written under apptainer.
RUN mkdir -p /var/lib/redis

# Pre-create bind-mount targets so apptainer/singularity has a real attach
# point. Docker auto-creates missing `-v` targets, but singularity uses a
# read-only underlay and silently ignores `:rw` when the target isn't a
# real directory in the SIF — writes then fail with EROFS even though the
# host bind path is writable. Pre-creating these directories costs one
# inode each and changes nothing in docker mode (the user's volume mount
# shadows them).
RUN mkdir -p /workspaces-streamlit-template /mounted-data

# Create workdir and copy over all streamlit related files/folders.

# note: specifying folder with slash as suffix and repeating the folder name seems important to preserve directory structure
WORKDIR /app
COPY assets/ /app/assets
COPY static/ /app/static
COPY content/ /app/content
COPY docs/ /app/docs
COPY example-data/ /app/example-data
COPY gdpr_consent/ /app/gdpr_consent
COPY hooks/ /app/hooks
COPY src/ /app/src
COPY utils/ /app/utils
COPY app.py /app/app.py
COPY settings.json /app/settings.json
COPY default-parameters.json /app/default-parameters.json
COPY presets.json /app/presets.json

# For streamlit configuration
COPY .streamlit/ /app/.streamlit/
COPY clean-up-workspaces.py /app/clean-up-workspaces.py

# Install openms-insight (built in the insight-build stage above).
# Only the package, not js-component/node_modules. .gitignore keeps hatch from
# adding the bundle twice (it is force-included because git ignores it).
COPY --from=insight-build /openms-insight/pyproject.toml /openms-insight/README.md /openms-insight/LICENSE /openms-insight/.gitignore /tmp/openms-insight/
COPY --from=insight-build /openms-insight/openms_insight /tmp/openms-insight/openms_insight
# Its dependencies come from requirements.txt; this replaces the PyPI release
RUN pip install --no-deps --force-reinstall /tmp/openms-insight && rm -rf /tmp/openms-insight

# add cron job to the crontab
RUN echo "0 3 * * * /root/miniforge3/envs/streamlit-env/bin/python /app/clean-up-workspaces.py >> /app/clean-up-workspaces.log 2>&1" | crontab -

# Set default worker count (can be overridden via environment variable)
ENV RQ_WORKER_COUNT=1
ENV REDIS_URL=redis://localhost:6379/0

# Number of Streamlit server instances for load balancing (default: 1 = no load balancer)
# Set to >1 to enable nginx load balancer with multiple Streamlit instances
ENV STREAMLIT_SERVER_COUNT=1

# Install the apptainer-compatible entrypoint that starts cron (when the root
# FS is writable), Redis, RQ workers, optional nginx load balancer, and the
# Streamlit server. The script falls back to /tmp paths under apptainer.
COPY docker/entrypoint.sh /app/entrypoint.sh
RUN chmod +x /app/entrypoint.sh

# Install the demo-seeding script run by the seed-demos initContainer
# (k8s/base/streamlit-deployment.yaml). One copy of the logic, so what the
# image ships is what tests/test_seed_demos.py executes.
COPY docker/seed-demos.sh /app/docker/seed-demos.sh
RUN chmod +x /app/docker/seed-demos.sh

# Patch Analytics
RUN mamba run -n streamlit-env python hooks/hook-analytics.py

# Set Online Deployment
RUN jq '.online_deployment = true' settings.json > tmp.json && mv tmp.json settings.json

# Point the in-app mounted-drive browser at the conventional bind-mount path.
# The browser only renders when this directory exists at runtime, i.e. when
# the user starts the container with `-v /host/path:/mounted-data`.
RUN jq '.local_data_dir = "/mounted-data"' settings.json > tmp.json && mv tmp.json settings.json

# Download the Windows installer of the latest release, if it has one.
# The token arrives as a BuildKit secret (`--secret id=github_token,env=GITHUB_TOKEN`),
# so it is never written to the image or its build cache. INSTALLER_REFRESH
# (CI passes the run id) re-runs this step on every build, as the token used to.
ARG INSTALLER_REFRESH
# Declared again: a build argument does not reach a later stage on its own.
ARG GITHUB_USER
ARG GITHUB_REPO
# FLASHApp: RELEASE_TAG pins the download to the release being published (set
# by the build-and-test workflow on release events). Empty: the latest release.
ARG RELEASE_TAG
RUN --mount=type=secret,id=github_token \
    repo="${GITHUB_USER:-$(jq -r '."github-user"' settings.json)}/${GITHUB_REPO:-$(jq -r '."repository-name"' settings.json)}"; \
    if [ -s /run/secrets/github_token ]; then \
        echo "GitHub token secret is set, proceeding to download the release asset..."; \
        if [ -n "$RELEASE_TAG" ]; then \
            GH_TOKEN="$(cat /run/secrets/github_token)" gh release download "$RELEASE_TAG" -R "$repo" -p "*-Setup.exe" -O /app/OpenMS-App-Setup.exe; \
        else \
            GH_TOKEN="$(cat /run/secrets/github_token)" gh release download -R "$repo" -p "*-Setup.exe" -O /app/OpenMS-App-Setup.exe \
                || echo "The latest release has no Windows installer, skipping."; \
        fi; \
    else \
        echo "No github_token secret, skipping the release asset download."; \
    fi


# Run app as container entrypoint.
EXPOSE $PORT
ENTRYPOINT ["/app/entrypoint.sh"]